You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2014/11/01 05:56:55 UTC

[01/50] [abbrv] Merge branch '1.5' into 1.6

Repository: accumulo
Updated Branches:
  refs/heads/1.5 69d665cb4 -> 2deabd31a
  refs/heads/1.6 5e0f6a426 -> 9b20a9d40
  refs/heads/master 64d1bbe9b -> cfbdef134


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
index c07ed1c,0000000..d77d060
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
@@@ -1,193 -1,0 +1,194 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.MemoryUnit;
 +import org.apache.accumulo.minicluster.ServerType;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.test.TestIngest;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.junit.Assert;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class LargeRowIT extends ConfigurableMacIT {
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 2, MemoryUnit.BYTE);
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "10ms"));
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 4 * 60;
 +  }
 +
 +  private static final int SEED = 42;
 +  private static final String REG_TABLE_NAME = "lr";
 +  private static final String PRE_SPLIT_TABLE_NAME = "lrps";
 +  private static final int NUM_ROWS = 100;
 +  private static final int ROW_SIZE = 1 << 17;
 +  private static final int NUM_PRE_SPLITS = 9;
 +  private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / NUM_PRE_SPLITS;
 +
 +  private int timeoutFactor = 1;
 +
 +  @Before
 +  public void getTimeoutFactor() {
 +    try {
 +      timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
 +    } catch (NumberFormatException e) {
 +      log.warn("Could not parse property value for 'timeout.factor' as integer: " + System.getProperty("timeout.factor"));
 +    }
 +
 +    Assert.assertTrue("Timeout factor must be greater than or equal to 1", timeoutFactor >= 1);
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Random r = new Random();
 +    byte rowData[] = new byte[ROW_SIZE];
 +    r.setSeed(SEED + 1);
 +    TreeSet<Text> splitPoints = new TreeSet<Text>();
 +    for (int i = 0; i < NUM_PRE_SPLITS; i++) {
 +      r.nextBytes(rowData);
 +      TestIngest.toPrintableChars(rowData);
 +      splitPoints.add(new Text(rowData));
 +    }
 +    Connector c = getConnector();
 +    c.tableOperations().create(REG_TABLE_NAME);
 +    c.tableOperations().create(PRE_SPLIT_TABLE_NAME);
 +    c.tableOperations().addSplits(PRE_SPLIT_TABLE_NAME, splitPoints);
 +    test1(c);
 +    test2(c);
 +  }
 +
 +  private void test1(Connector c) throws Exception {
 +
 +    basicTest(c, REG_TABLE_NAME, 0);
 +
 +    c.tableOperations().setProperty(REG_TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "" + SPLIT_THRESH);
 +
 +    UtilWaitThread.sleep(timeoutFactor * 12000);
 +    Logger.getLogger(LargeRowIT.class).warn("checking splits");
 +    FunctionalTestUtils.checkSplits(c, REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
 +
 +    verify(c, REG_TABLE_NAME);
 +  }
 +
 +  private void test2(Connector c) throws Exception {
 +    basicTest(c, PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
 +  }
 +
 +  private void basicTest(Connector c, String table, int expectedSplits) throws Exception {
 +    BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
 +
 +    Random r = new Random();
 +    byte rowData[] = new byte[ROW_SIZE];
 +
 +    r.setSeed(SEED);
 +
 +    for (int i = 0; i < NUM_ROWS; i++) {
 +
 +      r.nextBytes(rowData);
 +      TestIngest.toPrintableChars(rowData);
 +
 +      Mutation mut = new Mutation(new Text(rowData));
-       mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(UTF_8)));
 +      bw.addMutation(mut);
 +    }
 +
 +    bw.close();
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +
 +    verify(c, table);
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +
 +    c.tableOperations().flush(table, null, null, false);
 +
 +    // verify while table flush is running
 +    verify(c, table);
 +
 +    // give split time to complete
 +    c.tableOperations().flush(table, null, null, true);
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +
 +    verify(c, table);
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +  }
 +
 +  private void verify(Connector c, String table) throws Exception {
 +    Random r = new Random();
 +    byte rowData[] = new byte[ROW_SIZE];
 +
 +    r.setSeed(SEED);
 +
 +    Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
 +
 +    for (int i = 0; i < NUM_ROWS; i++) {
 +
 +      r.nextBytes(rowData);
 +      TestIngest.toPrintableChars(rowData);
 +
 +      scanner.setRange(new Range(new Text(rowData)));
 +
 +      int count = 0;
 +
 +      for (Entry<Key,Value> entry : scanner) {
 +        if (!entry.getKey().getRow().equals(new Text(rowData))) {
 +          throw new Exception("verification failed, unexpected row i =" + i);
 +        }
-         if (!entry.getValue().equals(Integer.toString(i).getBytes(Constants.UTF8))) {
++        if (!entry.getValue().equals(Integer.toString(i).getBytes(UTF_8))) {
 +          throw new Exception("verification failed, unexpected value i =" + i + " value = " + entry.getValue());
 +        }
 +        count++;
 +      }
 +
 +      if (count != 1) {
 +        throw new Exception("verification failed, unexpected count i =" + i + " count=" + count);
 +      }
 +
 +    }
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 3102f59,0000000..f4bee97
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -1,612 -1,0 +1,613 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.File;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.NoSuchElementException;
 +import java.util.Random;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.tserver.NativeMap;
 +import org.apache.hadoop.io.Text;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +public class NativeMapIT {
 +
 +  private Key nk(int r) {
 +    return new Key(new Text(String.format("r%09d", r)));
 +  }
 +
 +  private Key nk(int r, int cf, int cq, int cv, int ts, boolean deleted) {
 +    Key k = new Key(new Text(String.format("r%09d", r)), new Text(String.format("cf%09d", cf)), new Text(String.format("cq%09d", cq)), new Text(String.format(
 +        "cv%09d", cv)), ts);
 +
 +    k.setDeleted(deleted);
 +
 +    return k;
 +  }
 +
 +  private Value nv(int v) {
-     return new Value(String.format("r%09d", v).getBytes(Constants.UTF8));
++    return new Value(String.format("r%09d", v).getBytes(UTF_8));
 +  }
 +
 +  public static File nativeMapLocation() {
 +    File projectDir = new File(System.getProperty("user.dir")).getParentFile();
 +    File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION);
 +    return nativeMapDir;
 +  }
 +
 +  @BeforeClass
 +  public static void setUp() {
 +    NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
 +  }
 +
 +  private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {
 +    for (int i = start; i <= end; i++) {
 +      assertTrue(iter.hasNext());
 +      Entry<Key,Value> entry = iter.next();
 +      assertEquals(nk(i), entry.getKey());
 +      assertEquals(nv(i + valueOffset), entry.getValue());
 +    }
 +
 +    assertFalse(iter.hasNext());
 +  }
 +
 +  private void insertAndVerify(NativeMap nm, int start, int end, int valueOffset) {
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i + valueOffset));
 +    }
 +
 +    for (int i = start; i <= end; i++) {
 +      Value v = nm.get(nk(i));
 +      assertNotNull(v);
 +      assertEquals(nv(i + valueOffset), v);
 +
 +      Iterator<Entry<Key,Value>> iter2 = nm.iterator(nk(i));
 +      assertTrue(iter2.hasNext());
 +      Entry<Key,Value> entry = iter2.next();
 +      assertEquals(nk(i), entry.getKey());
 +      assertEquals(nv(i + valueOffset), entry.getValue());
 +    }
 +
 +    assertNull(nm.get(nk(start - 1)));
 +
 +    assertNull(nm.get(nk(end + 1)));
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    verifyIterator(start, end, valueOffset, iter);
 +
 +    for (int i = start; i <= end; i++) {
 +      iter = nm.iterator(nk(i));
 +      verifyIterator(i, end, valueOffset, iter);
 +
 +      // lookup nonexistant key that falls after existing key
 +      iter = nm.iterator(nk(i, 1, 1, 1, 1, false));
 +      verifyIterator(i + 1, end, valueOffset, iter);
 +    }
 +
 +    assertEquals(end - start + 1, nm.size());
 +  }
 +
 +  private void insertAndVerifyExhaustive(NativeMap nm, int num, int run) {
 +    for (int i = 0; i < num; i++) {
 +      for (int j = 0; j < num; j++) {
 +        for (int k = 0; k < num; k++) {
 +          for (int l = 0; l < num; l++) {
 +            for (int ts = 0; ts < num; ts++) {
 +              Key key = nk(i, j, k, l, ts, true);
-               Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++              Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
 +
 +              nm.put(key, value);
 +
 +              key = nk(i, j, k, l, ts, false);
-               value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++              value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
 +
 +              nm.put(key, value);
 +            }
 +          }
 +        }
 +      }
 +    }
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +
 +    for (int i = 0; i < num; i++) {
 +      for (int j = 0; j < num; j++) {
 +        for (int k = 0; k < num; k++) {
 +          for (int l = 0; l < num; l++) {
 +            for (int ts = num - 1; ts >= 0; ts--) {
 +              Key key = nk(i, j, k, l, ts, true);
-               Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++              Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
 +
 +              assertTrue(iter.hasNext());
 +              Entry<Key,Value> entry = iter.next();
 +              assertEquals(key, entry.getKey());
 +              assertEquals(value, entry.getValue());
 +
 +              key = nk(i, j, k, l, ts, false);
-               value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++              value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
 +
 +              assertTrue(iter.hasNext());
 +              entry = iter.next();
 +              assertEquals(key, entry.getKey());
 +              assertEquals(value, entry.getValue());
 +            }
 +          }
 +        }
 +      }
 +    }
 +
 +    assertFalse(iter.hasNext());
 +
 +    for (int i = 0; i < num; i++) {
 +      for (int j = 0; j < num; j++) {
 +        for (int k = 0; k < num; k++) {
 +          for (int l = 0; l < num; l++) {
 +            for (int ts = 0; ts < num; ts++) {
 +              Key key = nk(i, j, k, l, ts, true);
-               Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++              Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
 +
 +              assertEquals(value, nm.get(key));
 +
 +              Iterator<Entry<Key,Value>> iter2 = nm.iterator(key);
 +              assertTrue(iter2.hasNext());
 +              Entry<Key,Value> entry = iter2.next();
 +              assertEquals(key, entry.getKey());
 +              assertEquals(value, entry.getValue());
 +
 +              key = nk(i, j, k, l, ts, false);
-               value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++              value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
 +
 +              assertEquals(value, nm.get(key));
 +
 +              Iterator<Entry<Key,Value>> iter3 = nm.iterator(key);
 +              assertTrue(iter3.hasNext());
 +              Entry<Key,Value> entry2 = iter3.next();
 +              assertEquals(key, entry2.getKey());
 +              assertEquals(value, entry2.getValue());
 +            }
 +          }
 +        }
 +      }
 +    }
 +
 +    assertEquals(num * num * num * num * num * 2, nm.size());
 +  }
 +
 +  @Test
 +  public void test1() {
 +    NativeMap nm = new NativeMap();
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    assertFalse(iter.hasNext());
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test2() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerify(nm, 1, 10, 0);
 +    insertAndVerify(nm, 1, 10, 1);
 +    insertAndVerify(nm, 1, 10, 2);
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test4() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerifyExhaustive(nm, 3, 0);
 +    insertAndVerifyExhaustive(nm, 3, 1);
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test5() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerify(nm, 1, 10, 0);
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    iter.next();
 +
 +    nm.delete();
 +
 +    try {
 +      nm.put(nk(1), nv(1));
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.get(nk(1));
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.iterator();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.iterator(nk(1));
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.size();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      iter.next();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +  }
 +
 +  @Test
 +  public void test7() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerify(nm, 1, 10, 0);
 +
 +    nm.delete();
 +
 +    try {
 +      nm.delete();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +  }
 +
 +  @Test
 +  public void test8() {
 +    // test verifies that native map sorts keys sharing some common prefix properly
 +
 +    NativeMap nm = new NativeMap();
 +
 +    TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
 +
 +    tm.put(new Key(new Text("fo")), new Value(new byte[] {'0'}));
 +    tm.put(new Key(new Text("foo")), new Value(new byte[] {'1'}));
 +    tm.put(new Key(new Text("foo1")), new Value(new byte[] {'2'}));
 +    tm.put(new Key(new Text("foo2")), new Value(new byte[] {'3'}));
 + 
 +    for (Entry<Key,Value> entry : tm.entrySet()) {
 +      nm.put(entry.getKey(), entry.getValue());
 +    }
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +
 +    for (Entry<Key,Value> entry : tm.entrySet()) {
 +      assertTrue(iter.hasNext());
 +      Entry<Key,Value> entry2 = iter.next();
 +
 +      assertEquals(entry.getKey(), entry2.getKey());
 +      assertEquals(entry.getValue(), entry2.getValue());
 +    }
 +
 +    assertFalse(iter.hasNext());
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test9() {
 +    NativeMap nm = new NativeMap();
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +
 +    try {
 +      iter.next();
 +      assertTrue(false);
 +    } catch (NoSuchElementException e) {
 +
 +    }
 +
 +    insertAndVerify(nm, 1, 1, 0);
 +
 +    iter = nm.iterator();
 +    iter.next();
 +
 +    try {
 +      iter.next();
 +      assertTrue(false);
 +    } catch (NoSuchElementException e) {
 +
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test10() {
 +    int start = 1;
 +    int end = 10000;
 +
 +    NativeMap nm = new NativeMap();
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i));
 +    }
 +
 +    long mem1 = nm.getMemoryUsed();
 +
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i));
 +    }
 +
 +    long mem2 = nm.getMemoryUsed();
 +
 +    if (mem1 != mem2) {
 +      throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem2);
 +    }
 +
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i));
 +    }
 +
 +    long mem3 = nm.getMemoryUsed();
 +
 +    if (mem1 != mem3) {
 +      throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem3);
 +    }
 +
 +    byte bigrow[] = new byte[1000000];
 +    byte bigvalue[] = new byte[bigrow.length];
 +
 +    for (int i = 0; i < bigrow.length; i++) {
 +      bigrow[i] = (byte) (0xff & (i % 256));
 +      bigvalue[i] = bigrow[i];
 +    }
 +
 +    nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
 +
 +    long mem4 = nm.getMemoryUsed();
 +
 +    Value val = nm.get(new Key(new Text(bigrow)));
 +    if (val == null || !val.equals(new Value(bigvalue))) {
 +      throw new RuntimeException("Did not get expected big value");
 +    }
 +
 +    nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
 +
 +    long mem5 = nm.getMemoryUsed();
 +
 +    if (mem4 != mem5) {
 +      throw new RuntimeException("Memory changed after inserting duplicate data " + mem4 + " " + mem5);
 +    }
 +
 +    val = nm.get(new Key(new Text(bigrow)));
 +    if (val == null || !val.equals(new Value(bigvalue))) {
 +      throw new RuntimeException("Did not get expected big value");
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  // random length random field
 +  private static byte[] rlrf(Random r, int maxLen) {
 +    int len = r.nextInt(maxLen);
 +
 +    byte f[] = new byte[len];
 +    r.nextBytes(f);
 +
 +    return f;
 +  }
 +
 +  @Test
 +  public void test11() {
 +    NativeMap nm = new NativeMap();
 +
 +    // insert things with varying field sizes and value sizes
 +
 +    // generate random data
 +    Random r = new Random(75);
 +
 +    ArrayList<Pair<Key,Value>> testData = new ArrayList<Pair<Key,Value>>();
 +
 +    for (int i = 0; i < 100000; i++) {
 +
 +      Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false);
 +      Value v = new Value(rlrf(r, 511));
 +
 +      testData.add(new Pair<Key,Value>(k, v));
 +    }
 +
 +    // insert unsorted data
 +    for (Pair<Key,Value> pair : testData) {
 +      nm.put(pair.getFirst(), pair.getSecond());
 +    }
 +
 +    for (int i = 0; i < 2; i++) {
 +
 +      // sort data
 +      Collections.sort(testData, new Comparator<Pair<Key,Value>>() {
 +        @Override
 +        public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
 +          return o1.getFirst().compareTo(o2.getFirst());
 +        }
 +      });
 +
 +      // verify
 +      Iterator<Entry<Key,Value>> iter1 = nm.iterator();
 +      Iterator<Pair<Key,Value>> iter2 = testData.iterator();
 +
 +      while (iter1.hasNext() && iter2.hasNext()) {
 +        Entry<Key,Value> e = iter1.next();
 +        Pair<Key,Value> p = iter2.next();
 +
 +        if (!e.getKey().equals(p.getFirst()))
 +          throw new RuntimeException("Keys not equal");
 +
 +        if (!e.getValue().equals(p.getSecond()))
 +          throw new RuntimeException("Values not equal");
 +      }
 +
 +      if (iter1.hasNext())
 +        throw new RuntimeException("Not all of native map consumed");
 +
 +      if (iter2.hasNext())
 +        throw new RuntimeException("Not all of test data consumed");
 +
 +      System.out.println("test 11 nm mem " + nm.getMemoryUsed());
 +
 +      // insert data again w/ different value
 +      Collections.shuffle(testData, r);
 +      // insert unsorted data
 +      for (Pair<Key,Value> pair : testData) {
 +        pair.getSecond().set(rlrf(r, 511));
 +        nm.put(pair.getFirst(), pair.getSecond());
 +      }
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void testBinary() {
 +    NativeMap nm = new NativeMap();
 +
 +    byte emptyBytes[] = new byte[0];
 +
 +    for (int i = 0; i < 256; i++) {
 +      for (int j = 0; j < 256; j++) {
 +        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
 +        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
 +
 +        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
 +        Value v = new Value(data);
 +
 +        nm.put(k, v);
 +      }
 +    }
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    for (int i = 0; i < 256; i++) {
 +      for (int j = 0; j < 256; j++) {
 +        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
 +        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
 +
 +        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
 +        Value v = new Value(data);
 +
 +        assertTrue(iter.hasNext());
 +        Entry<Key,Value> entry = iter.next();
 +
 +        assertEquals(k, entry.getKey());
 +        assertEquals(v, entry.getValue());
 +
 +      }
 +    }
 +
 +    assertFalse(iter.hasNext());
 +
 +    for (int i = 0; i < 256; i++) {
 +      for (int j = 0; j < 256; j++) {
 +        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
 +        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
 +
 +        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
 +        Value v = new Value(data);
 +
 +        Value v2 = nm.get(k);
 +
 +        assertEquals(v, v2);
 +      }
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void testEmpty() {
 +    NativeMap nm = new NativeMap();
 +
 +    assertTrue(nm.size() == 0);
 +    assertTrue(nm.getMemoryUsed() == 0);
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void testConcurrentIter() throws IOException {
 +    NativeMap nm = new NativeMap();
 +
 +    nm.put(nk(0), nv(0));
 +    nm.put(nk(1), nv(1));
 +    nm.put(nk(3), nv(3));
 +
 +    SortedKeyValueIterator<Key,Value> iter = nm.skvIterator();
 +
 +    // modify map after iter created
 +    nm.put(nk(2), nv(2));
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(0));
 +    iter.next();
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(1));
 +    iter.next();
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(2));
 +    iter.next();
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(3));
 +    iter.next();
 +
 +    assertFalse(iter.hasTop());
 +
 +    nm.delete();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index 7e43812,0000000..189a55c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@@ -1,132 -1,0 +1,133 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.ScannerBase;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ScanIteratorIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 30;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    String tableName = getUniqueNames(1)[0];
 +    Connector c = getConnector();
 +    c.tableOperations().create(tableName);
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    for (int i = 0; i < 1000; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%06d", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(Constants.UTF8)));
-       m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(UTF_8)));
++      m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(UTF_8)));
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +
 +    Scanner scanner = c.createScanner(tableName, new Authorizations());
 +
 +    setupIter(scanner);
 +    verify(scanner, 1, 999);
 +
 +    BatchScanner bscanner = c.createBatchScanner(tableName, new Authorizations(), 3);
 +    bscanner.setRanges(Collections.singleton(new Range((Key) null, null)));
 +
 +    setupIter(bscanner);
 +    verify(bscanner, 1, 999);
 +
 +    ArrayList<Range> ranges = new ArrayList<Range>();
 +    ranges.add(new Range(new Text(String.format("%06d", 1))));
 +    ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16))));
 +    ranges.add(new Range(new Text(String.format("%06d", 20))));
 +    ranges.add(new Range(new Text(String.format("%06d", 23))));
 +    ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61))));
 +    ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504))));
 +    ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000))));
 +
 +    HashSet<Integer> got = new HashSet<Integer>();
 +    HashSet<Integer> expected = new HashSet<Integer>();
 +    for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) {
 +      expected.add(i);
 +    }
 +
 +    bscanner.setRanges(ranges);
 +
 +    for (Entry<Key,Value> entry : bscanner) {
 +      got.add(Integer.parseInt(entry.getKey().getRow().toString()));
 +    }
 +
 +    System.out.println("got : " + got);
 +
 +    if (!got.equals(expected)) {
 +      throw new Exception(got + " != " + expected);
 +    }
 +
 +    bscanner.close();
 +
 +  }
 +
 +  private void verify(Iterable<Entry<Key,Value>> scanner, int start, int finish) throws Exception {
 +
 +    int expected = start;
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (Integer.parseInt(entry.getKey().getRow().toString()) != expected) {
 +        throw new Exception("Saw unexpexted " + entry.getKey().getRow() + " " + expected);
 +      }
 +
 +      if (entry.getKey().getColumnQualifier().toString().equals("cq2")) {
 +        expected += 2;
 +      }
 +    }
 +
 +    if (expected != finish + 2) {
 +      throw new Exception("Ended at " + expected + " not " + (finish + 2));
 +    }
 +  }
 +
 +  private void setupIter(ScannerBase scanner) throws Exception {
 +    IteratorSetting dropMod = new IteratorSetting(50, "dropMod", "org.apache.accumulo.test.functional.DropModIter");
 +    dropMod.addOption("mod", "2");
 +    dropMod.addOption("drop", "0");
 +    scanner.addScanIterator(dropMod);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
index ba03d00,0000000..90b881c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
@@@ -1,242 -1,0 +1,243 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Map.Entry;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ScanRangeIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  private static final int TS_LIMIT = 1;
 +  private static final int CQ_LIMIT = 5;
 +  private static final int CF_LIMIT = 5;
 +  private static final int ROW_LIMIT = 100;
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    String[] tableNames = getUniqueNames(2);
 +    String table1 = tableNames[0];
 +    c.tableOperations().create(table1);
 +    String table2 = tableNames[1];
 +    c.tableOperations().create(table2);
 +    TreeSet<Text> splitRows = new TreeSet<Text>();
 +    int splits = 3;
 +    for (int i = (ROW_LIMIT / splits); i < ROW_LIMIT; i += (ROW_LIMIT / splits))
 +      splitRows.add(createRow(i));
 +    c.tableOperations().addSplits(table2, splitRows);
 +
 +    insertData(c, table1);
 +    scanTable(c, table1);
 +
 +    insertData(c, table2);
 +    scanTable(c, table2);
 +  }
 +
 +  private void scanTable(Connector c, String table) throws Exception {
 +    scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(1, 0, 0, 0));
 +
 +    scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +
 +    scanRange(c, table, null, null);
 +
 +    for (int i = 0; i < ROW_LIMIT; i += (ROW_LIMIT / 3)) {
 +      for (int j = 0; j < CF_LIMIT; j += (CF_LIMIT / 2)) {
 +        for (int k = 1; k < CQ_LIMIT; k += (CQ_LIMIT / 2)) {
 +          scanRange(c, table, null, new IntKey(i, j, k, 0));
 +          scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(i, j, k, 0));
 +
 +          scanRange(c, table, new IntKey(i, j, k, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +
 +          scanRange(c, table, new IntKey(i, j, k, 0), null);
 +
 +        }
 +      }
 +    }
 +
 +    for (int i = 0; i < ROW_LIMIT; i++) {
 +      scanRange(c, table, new IntKey(i, 0, 0, 0), new IntKey(i, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +
 +      if (i > 0 && i < ROW_LIMIT - 1) {
 +        scanRange(c, table, new IntKey(i - 1, 0, 0, 0), new IntKey(i + 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +      }
 +    }
 +
 +  }
 +
 +  private static class IntKey {
 +    private int row;
 +    private int cf;
 +    private int cq;
 +    private long ts;
 +
 +    IntKey(IntKey ik) {
 +      this.row = ik.row;
 +      this.cf = ik.cf;
 +      this.cq = ik.cq;
 +      this.ts = ik.ts;
 +    }
 +
 +    IntKey(int row, int cf, int cq, long ts) {
 +      this.row = row;
 +      this.cf = cf;
 +      this.cq = cq;
 +      this.ts = ts;
 +    }
 +
 +    Key createKey() {
 +      Text trow = createRow(row);
 +      Text tcf = createCF(cf);
 +      Text tcq = createCQ(cq);
 +
 +      return new Key(trow, tcf, tcq, ts);
 +    }
 +
 +    IntKey increment() {
 +
 +      IntKey ik = new IntKey(this);
 +
 +      ik.ts++;
 +      if (ik.ts >= TS_LIMIT) {
 +        ik.ts = 0;
 +        ik.cq++;
 +        if (ik.cq >= CQ_LIMIT) {
 +          ik.cq = 0;
 +          ik.cf++;
 +          if (ik.cf >= CF_LIMIT) {
 +            ik.cf = 0;
 +            ik.row++;
 +          }
 +        }
 +      }
 +
 +      return ik;
 +    }
 +
 +  }
 +
 +  private void scanRange(Connector c, String table, IntKey ik1, IntKey ik2) throws Exception {
 +    scanRange(c, table, ik1, false, ik2, false);
 +    scanRange(c, table, ik1, false, ik2, true);
 +    scanRange(c, table, ik1, true, ik2, false);
 +    scanRange(c, table, ik1, true, ik2, true);
 +  }
 +
 +  private void scanRange(Connector c, String table, IntKey ik1, boolean inclusive1, IntKey ik2, boolean inclusive2) throws Exception {
 +    Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
 +
 +    Key key1 = null;
 +    Key key2 = null;
 +
 +    IntKey expectedIntKey;
 +    IntKey expectedEndIntKey;
 +
 +    if (ik1 != null) {
 +      key1 = ik1.createKey();
 +      expectedIntKey = ik1;
 +
 +      if (!inclusive1) {
 +        expectedIntKey = expectedIntKey.increment();
 +      }
 +    } else {
 +      expectedIntKey = new IntKey(0, 0, 0, 0);
 +    }
 +
 +    if (ik2 != null) {
 +      key2 = ik2.createKey();
 +      expectedEndIntKey = ik2;
 +
 +      if (inclusive2) {
 +        expectedEndIntKey = expectedEndIntKey.increment();
 +      }
 +    } else {
 +      expectedEndIntKey = new IntKey(ROW_LIMIT, 0, 0, 0);
 +    }
 +
 +    Range range = new Range(key1, inclusive1, key2, inclusive2);
 +
 +    scanner.setRange(range);
 +
 +    for (Entry<Key,Value> entry : scanner) {
 +
 +      Key expectedKey = expectedIntKey.createKey();
 +      if (!expectedKey.equals(entry.getKey())) {
 +        throw new Exception(" " + expectedKey + " != " + entry.getKey());
 +      }
 +
 +      expectedIntKey = expectedIntKey.increment();
 +    }
 +
 +    if (!expectedIntKey.createKey().equals(expectedEndIntKey.createKey())) {
 +      throw new Exception(" " + expectedIntKey.createKey() + " != " + expectedEndIntKey.createKey());
 +    }
 +  }
 +
 +  private static Text createCF(int cf) {
 +    Text tcf = new Text(String.format("cf_%03d", cf));
 +    return tcf;
 +  }
 +
 +  private static Text createCQ(int cf) {
 +    Text tcf = new Text(String.format("cq_%03d", cf));
 +    return tcf;
 +  }
 +
 +  private static Text createRow(int row) {
 +    Text trow = new Text(String.format("r_%06d", row));
 +    return trow;
 +  }
 +
 +  private void insertData(Connector c, String table) throws Exception {
 +
 +    BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
 +
 +    for (int i = 0; i < ROW_LIMIT; i++) {
 +      Mutation m = new Mutation(createRow(i));
 +
 +      for (int j = 0; j < CF_LIMIT; j++) {
 +        for (int k = 0; k < CQ_LIMIT; k++) {
 +          for (int t = 0; t < TS_LIMIT; t++) {
-             m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(Constants.UTF8)));
++            m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(UTF_8)));
 +          }
 +        }
 +      }
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
index ed2f0ba,0000000..3547b68
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
@@@ -1,111 -1,0 +1,112 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ScanSessionTimeOutIT extends ConfigurableMacIT {
 +  
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_SESSION_MAXIDLE.getKey(), "3"));
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    c.tableOperations().create("abc");
 +    
 +    BatchWriter bw = c.createBatchWriter("abc", new BatchWriterConfig());
 +    
 +    for (int i = 0; i < 100000; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%08d", i)));
 +      for (int j = 0; j < 3; j++)
-         m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(Constants.UTF8)));
++        m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
 +      
 +      bw.addMutation(m);
 +    }
 +    
 +    bw.close();
 +    
 +    Scanner scanner = c.createScanner("abc", new Authorizations());
 +    scanner.setBatchSize(1000);
 +    
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +    
 +    verify(iter, 0, 200);
 +    
 +    // sleep three times the session timeout
 +    UtilWaitThread.sleep(9000);
 +    
 +    verify(iter, 200, 100000);
 +    
 +  }
 +  
 +  private void verify(Iterator<Entry<Key,Value>> iter, int start, int stop) throws Exception {
 +    for (int i = start; i < stop; i++) {
 +      
 +      Text er = new Text(String.format("%08d", i));
 +      
 +      for (int j = 0; j < 3; j++) {
 +        Entry<Key,Value> entry = iter.next();
 +        
 +        if (!entry.getKey().getRow().equals(er)) {
 +          throw new Exception("row " + entry.getKey().getRow() + " != " + er);
 +        }
 +        
 +        if (!entry.getKey().getColumnFamily().equals(new Text("cf1"))) {
 +          throw new Exception("cf " + entry.getKey().getColumnFamily() + " != cf1");
 +        }
 +        
 +        if (!entry.getKey().getColumnQualifier().equals(new Text("cq" + j))) {
 +          throw new Exception("cq " + entry.getKey().getColumnQualifier() + " != cq" + j);
 +        }
 +        
 +        if (!entry.getValue().toString().equals("" + i + "_" + j)) {
 +          throw new Exception("value " + entry.getValue() + " != " + i + "_" + j);
 +        }
 +        
 +      }
 +    }
 +    
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index ea92949,0000000..43d3a86
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@@ -1,272 -1,0 +1,273 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.ArrayList;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Writer;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.rfile.RFile;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.master.state.Assignment;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.util.FileUtil;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.tserver.TabletServer;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class SplitRecoveryIT extends ConfigurableMacIT {
 +  
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +  
 +  private KeyExtent nke(String table, String endRow, String prevEndRow) {
 +    return new KeyExtent(new Text(table), endRow == null ? null : new Text(endRow), prevEndRow == null ? null : new Text(prevEndRow));
 +  }
 +  
 +  private void run() throws Exception {
 +    String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/testLock";
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.OVERWRITE);
 +    ZooLock zl = new ZooLock(zPath);
 +    boolean gotLock = zl.tryLock(new LockWatcher() {
 +      
 +      @Override
 +      public void lostLock(LockLossReason reason) {
 +        System.exit(-1);
 +        
 +      }
 +      
 +      @Override
 +      public void unableToMonitorLockNode(Throwable e) {
 +        System.exit(-1);
 +      }
-     }, "foo".getBytes(Constants.UTF8));
++    }, "foo".getBytes(UTF_8));
 +    
 +    if (!gotLock) {
 +      System.err.println("Failed to get lock " + zPath);
 +    }
 +    
 +    // run test for a table with one tablet
 +    runSplitRecoveryTest(0, "sp", 0, zl, nke("foo0", null, null));
 +    runSplitRecoveryTest(1, "sp", 0, zl, nke("foo1", null, null));
 +    
 +    // run test for tables with two tablets, run test on first and last tablet
 +    runSplitRecoveryTest(0, "k", 0, zl, nke("foo2", "m", null), nke("foo2", null, "m"));
 +    runSplitRecoveryTest(1, "k", 0, zl, nke("foo3", "m", null), nke("foo3", null, "m"));
 +    runSplitRecoveryTest(0, "o", 1, zl, nke("foo4", "m", null), nke("foo4", null, "m"));
 +    runSplitRecoveryTest(1, "o", 1, zl, nke("foo5", "m", null), nke("foo5", null, "m"));
 +    
 +    // run test for table w/ three tablets, run test on middle tablet
 +    runSplitRecoveryTest(0, "o", 1, zl, nke("foo6", "m", null), nke("foo6", "r", "m"), nke("foo6", null, "r"));
 +    runSplitRecoveryTest(1, "o", 1, zl, nke("foo7", "m", null), nke("foo7", "r", "m"), nke("foo7", null, "r"));
 +    
 +    // run test for table w/ three tablets, run test on first
 +    runSplitRecoveryTest(0, "g", 0, zl, nke("foo8", "m", null), nke("foo8", "r", "m"), nke("foo8", null, "r"));
 +    runSplitRecoveryTest(1, "g", 0, zl, nke("foo9", "m", null), nke("foo9", "r", "m"), nke("foo9", null, "r"));
 +    
 +    // run test for table w/ three tablets, run test on last tablet
 +    runSplitRecoveryTest(0, "w", 2, zl, nke("fooa", "m", null), nke("fooa", "r", "m"), nke("fooa", null, "r"));
 +    runSplitRecoveryTest(1, "w", 2, zl, nke("foob", "m", null), nke("foob", "r", "m"), nke("foob", null, "r"));
 +  }
 +  
 +  private void runSplitRecoveryTest(int failPoint, String mr, int extentToSplit, ZooLock zl, KeyExtent... extents) throws Exception {
 +    
 +    Text midRow = new Text(mr);
 +    
 +    SortedMap<FileRef,DataFileValue> splitMapFiles = null;
 +    
 +    for (int i = 0; i < extents.length; i++) {
 +      KeyExtent extent = extents[i];
 +      
 +      String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString() + "/dir_" + i;
 +      MetadataTableUtil.addTablet(extent, tdir, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID, zl);
 +      SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>();
 +      mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i));
 +      
 +      if (i == extentToSplit) {
 +        splitMapFiles = mapFiles;
 +      }
 +      int tid = 0;
 +      TransactionWatcher.ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
 +      MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles, "L0", SystemCredentials.get(), zl);
 +    }
 +    
 +    KeyExtent extent = extents[extentToSplit];
 +    
 +    KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
 +    KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
 +    
 +    splitPartiallyAndRecover(extent, high, low, .4, splitMapFiles, midRow, "localhost:1234", failPoint, zl);
 +  }
 +  
 +  private void splitPartiallyAndRecover(KeyExtent extent, KeyExtent high, KeyExtent low, double splitRatio, SortedMap<FileRef,DataFileValue> mapFiles,
 +      Text midRow, String location, int steps, ZooLock zl) throws Exception {
 +    
 +    SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>();
 +    SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
 +    List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
 +    
 +    MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, new HashMap<FileRef,FileUtil.FileInfo>(), mapFiles, lowDatafileSizes,
 +        highDatafileSizes, highDatafilesToRemove);
 +    
 +    MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, SystemCredentials.get(), zl);
 +    TServerInstance instance = new TServerInstance(location, zl.getSessionId());
 +    Writer writer = new Writer(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID);
 +    Assignment assignment = new Assignment(high, instance);
 +    Mutation m = new Mutation(assignment.tablet.getMetadataEntry());
 +    m.put(TabletsSection.FutureLocationColumnFamily.NAME, assignment.server.asColumnQualifier(), assignment.server.asMutationValue());
 +    writer.update(m);
 +    
 +    if (steps >= 1) {
 +      Map<FileRef,Long> bulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), extent);
 +      MasterMetadataUtil.addNewTablet(low, "/lowDir", instance, lowDatafileSizes, bulkFiles, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID + "0", -1l,
 +          -1l, zl);
 +    }
 +    if (steps >= 2)
 +      MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SystemCredentials.get(), zl);
 +    
 +    TabletServer.verifyTabletInformation(high, instance, null, "127.0.0.1:0", zl);
 +    
 +    if (steps >= 1) {
 +      ensureTabletHasNoUnexpectedMetadataEntries(low, lowDatafileSizes);
 +      ensureTabletHasNoUnexpectedMetadataEntries(high, highDatafileSizes);
 +      
 +      Map<FileRef,Long> lowBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), low);
 +      Map<FileRef,Long> highBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), high);
 +      
 +      if (!lowBulkFiles.equals(highBulkFiles)) {
 +        throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
 +      }
 +      
 +      if (lowBulkFiles.size() == 0) {
 +        throw new Exception(" no bulk files " + low);
 +      }
 +    } else {
 +      ensureTabletHasNoUnexpectedMetadataEntries(extent, mapFiles);
 +    }
 +  }
 +  
 +  private void ensureTabletHasNoUnexpectedMetadataEntries(KeyExtent extent, SortedMap<FileRef,DataFileValue> expectedMapFiles) throws Exception {
 +    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(extent.toMetadataRange());
 +    
 +    HashSet<ColumnFQ> expectedColumns = new HashSet<ColumnFQ>();
 +    expectedColumns.add(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN);
 +    expectedColumns.add(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
 +    expectedColumns.add(TabletsSection.ServerColumnFamily.TIME_COLUMN);
 +    expectedColumns.add(TabletsSection.ServerColumnFamily.LOCK_COLUMN);
 +    
 +    HashSet<Text> expectedColumnFamilies = new HashSet<Text>();
 +    expectedColumnFamilies.add(DataFileColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.FutureLocationColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.CurrentLocationColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.LastLocationColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.BulkFileColumnFamily.NAME);
 +    
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +    while (iter.hasNext()) {
 +      Key key = iter.next().getKey();
 +      
 +      if (!key.getRow().equals(extent.getMetadataEntry())) {
 +        throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
 +      }
 +      
 +      if (expectedColumnFamilies.contains(key.getColumnFamily())) {
 +        continue;
 +      }
 +      
 +      if (expectedColumns.remove(new ColumnFQ(key))) {
 +        continue;
 +      }
 +      
 +      throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
 +    }
 +    System.out.println("expectedColumns " + expectedColumns);
 +    if (expectedColumns.size() > 1 || (expectedColumns.size() == 1)) {
 +      throw new Exception("Not all expected columns seen " + extent + " " + expectedColumns);
 +    }
 +    
 +    SortedMap<FileRef,DataFileValue> fixedMapFiles = MetadataTableUtil.getDataFileSizes(extent, SystemCredentials.get());
 +    verifySame(expectedMapFiles, fixedMapFiles);
 +  }
 +  
 +  private void verifySame(SortedMap<FileRef,DataFileValue> datafileSizes, SortedMap<FileRef,DataFileValue> fixedDatafileSizes) throws Exception {
 +    
 +    if (!datafileSizes.keySet().containsAll(fixedDatafileSizes.keySet()) || !fixedDatafileSizes.keySet().containsAll(datafileSizes.keySet())) {
 +      throw new Exception("Key sets not the same " + datafileSizes.keySet() + " !=  " + fixedDatafileSizes.keySet());
 +    }
 +    
 +    for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
 +      DataFileValue dfv = entry.getValue();
 +      DataFileValue otherDfv = fixedDatafileSizes.get(entry.getKey());
 +      
 +      if (!dfv.equals(otherDfv)) {
 +        throw new Exception(entry.getKey() + " dfv not equal  " + dfv + "  " + otherDfv);
 +      }
 +    }
 +  }
 +
 +  
 +  public static void main(String[] args) throws Exception {
 +    new SplitRecoveryIT().run();
 +  }
 +  
 +  @Test
 +  public void test() throws Exception {
 +    assertEquals(0, exec(SplitRecoveryIT.class).waitFor());
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
index b74399d,0000000..fccc79f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
@@@ -1,101 -1,0 +1,101 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.minicluster.MemoryUnit;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class TabletIT extends ConfigurableMacIT {
 +
 +  private static final int N = 1000;
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    Map<String,String> siteConfig = new HashMap<String,String>();
 +    siteConfig.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "200");
 +    siteConfig.put(Property.TSERV_MAXMEM.getKey(), "128M");
 +    cfg.setDefaultMemory(256, MemoryUnit.MEGABYTE);
 +    cfg.setSiteConfig(siteConfig);
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void createTableTest() throws Exception {
 +    String tableName = getUniqueNames(1)[0];
 +    createTableTest(tableName, false);
 +    createTableTest(tableName, true);
 +  }
 +
 +  public void createTableTest(String tableName, boolean readOnly) throws Exception {
 +    // create the test table within accumulo
 +    Connector connector = getConnector();
 +
 +    if (!readOnly) {
 +      TreeSet<Text> keys = new TreeSet<Text>();
 +      for (int i = N / 100; i < N; i += N / 100) {
 +        keys.add(new Text(String.format("%05d", i)));
 +      }
 +
 +      // presplit
 +      connector.tableOperations().create(tableName);
 +      connector.tableOperations().addSplits(tableName, keys);
 +      BatchWriter b = connector.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +      // populate
 +      for (int i = 0; i < N; i++) {
 +        Mutation m = new Mutation(new Text(String.format("%05d", i)));
-         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
++        m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
 +        b.addMutation(m);
 +      }
 +      b.close();
 +    }
 +
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +    int count = 0;
 +    for (Entry<Key,Value> elt : scanner) {
 +      String expected = String.format("%05d", count);
 +      assert (elt.getKey().getRow().toString().equals(expected));
 +      count++;
 +    }
 +    assertEquals(N, count);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
index 1dd0bf1,0000000..f2460cf
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@@ -1,303 -1,0 +1,304 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.util.ByteArraySet;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class VisibilityIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    String[] tableNames = getUniqueNames(2);
 +    String table = tableNames[0];
 +    c.tableOperations().create(table);
 +    String table2 = tableNames[1];
 +    c.tableOperations().create(table2);
 +    c.tableOperations().setProperty(table2, Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "DEFLABEL");
 +
 +    insertData(c, table);
 +    queryData(c, table);
 +    deleteData(c, table);
 +
 +    insertDefaultData(c, table2);
 +    queryDefaultData(c, table2);
 +
 +  }
 +
 +  private static SortedSet<String> nss(String... labels) {
 +    TreeSet<String> ts = new TreeSet<String>();
 +
 +    for (String s : labels) {
 +      ts.add(s);
 +    }
 +
 +    return ts;
 +  }
 +
 +  private void mput(Mutation m, String cf, String cq, String cv, String val) {
-     ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
-     m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(Constants.UTF8)));
++    ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
++    m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(UTF_8)));
 +  }
 +
 +  private void mputDelete(Mutation m, String cf, String cq, String cv) {
-     ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
++    ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
 +    m.putDelete(new Text(cf), new Text(cq), le);
 +  }
 +
 +  private void insertData(Connector c, String tableName) throws Exception {
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m1 = new Mutation(new Text("row1"));
 +
 +    mput(m1, "cf1", "cq1", "", "v1");
 +    mput(m1, "cf1", "cq1", "A", "v2");
 +    mput(m1, "cf1", "cq1", "B", "v3");
 +    mput(m1, "cf1", "cq1", "A&B", "v4");
 +    mput(m1, "cf1", "cq1", "A&(L|M)", "v5");
 +    mput(m1, "cf1", "cq1", "B&(L|M)", "v6");
 +    mput(m1, "cf1", "cq1", "A&B&(L|M)", "v7");
 +    mput(m1, "cf1", "cq1", "A&B&(L)", "v8");
 +    mput(m1, "cf1", "cq1", "A&FOO", "v9");
 +    mput(m1, "cf1", "cq1", "A&FOO&(L|M)", "v10");
 +    mput(m1, "cf1", "cq1", "FOO", "v11");
 +    mput(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)", "v12");
 +    mput(m1, "cf1", "cq1", "A&B&(L|M|FOO)", "v13");
 +
 +    bw.addMutation(m1);
 +    bw.close();
 +  }
 +
 +  private void deleteData(Connector c, String tableName) throws Exception {
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m1 = new Mutation(new Text("row1"));
 +
 +    mputDelete(m1, "cf1", "cq1", "");
 +    mputDelete(m1, "cf1", "cq1", "A");
 +    mputDelete(m1, "cf1", "cq1", "A&B");
 +    mputDelete(m1, "cf1", "cq1", "B&(L|M)");
 +    mputDelete(m1, "cf1", "cq1", "A&B&(L)");
 +    mputDelete(m1, "cf1", "cq1", "A&FOO&(L|M)");
 +    mputDelete(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)");
 +    mputDelete(m1, "cf1", "cq1", "FOO&A"); // should not delete anything
 +
 +    bw.addMutation(m1);
 +    bw.close();
 +
 +    Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
 +
 +    expected.put(nss("A", "L"), nss("v5"));
 +    expected.put(nss("A", "M"), nss("v5"));
 +    expected.put(nss("B"), nss("v3"));
 +    expected.put(nss("Z"), nss());
 +    expected.put(nss("A", "B", "L"), nss("v7", "v13"));
 +    expected.put(nss("A", "B", "M"), nss("v7", "v13"));
 +    expected.put(nss("A", "B", "FOO"), nss("v13"));
 +    expected.put(nss("FOO"), nss("v11"));
 +    expected.put(nss("A", "FOO"), nss("v9"));
 +
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
 +  }
 +
 +  private void insertDefaultData(Connector c, String tableName) throws Exception {
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m1 = new Mutation(new Text("row1"));
 +
 +    mput(m1, "cf1", "cq1", "BASE", "v1");
 +    mput(m1, "cf1", "cq2", "DEFLABEL", "v2");
 +    mput(m1, "cf1", "cq3", "", "v3");
 +
 +    bw.addMutation(m1);
 +    bw.close();
 +  }
 +
 +  private static void uniqueCombos(List<Set<String>> all, Set<String> prefix, Set<String> suffix) {
 +
 +    all.add(prefix);
 +
 +    TreeSet<String> ss = new TreeSet<String>(suffix);
 +
 +    for (String s : suffix) {
 +      TreeSet<String> ps = new TreeSet<String>(prefix);
 +      ps.add(s);
 +      ss.remove(s);
 +
 +      uniqueCombos(all, ps, ss);
 +    }
 +  }
 +
 +  private void queryData(Connector c, String tableName) throws Exception {
 +    Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
 +    expected.put(nss(), nss("v1"));
 +    expected.put(nss("A"), nss("v2"));
 +    expected.put(nss("A", "L"), nss("v5"));
 +    expected.put(nss("A", "M"), nss("v5"));
 +    expected.put(nss("B"), nss("v3"));
 +    expected.put(nss("B", "L"), nss("v6"));
 +    expected.put(nss("B", "M"), nss("v6"));
 +    expected.put(nss("Z"), nss());
 +    expected.put(nss("A", "B"), nss("v4"));
 +    expected.put(nss("A", "B", "L"), nss("v7", "v8", "v13"));
 +    expected.put(nss("A", "B", "M"), nss("v7", "v13"));
 +    expected.put(nss("A", "B", "FOO"), nss("v13"));
 +    expected.put(nss("FOO"), nss("v11"));
 +    expected.put(nss("A", "FOO"), nss("v9"));
 +    expected.put(nss("A", "FOO", "L"), nss("v10", "v12"));
 +    expected.put(nss("A", "FOO", "M"), nss("v10", "v12"));
 +    expected.put(nss("B", "FOO", "L"), nss("v12"));
 +    expected.put(nss("B", "FOO", "M"), nss("v12"));
 +
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "L", "M", "Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected);
 +  }
 +
 +  private void queryData(Connector c, String tableName, Set<String> allAuths, Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws Exception {
 +
 +    c.securityOperations().changeUserAuthorizations("root", new Authorizations(nbas(userAuths)));
 +
 +    ArrayList<Set<String>> combos = new ArrayList<Set<String>>();
 +    uniqueCombos(combos, nss(), allAuths);
 +
 +    for (Set<String> set1 : combos) {
 +      Set<String> e = new TreeSet<String>();
 +      for (Set<String> set2 : combos) {
 +
 +        set2 = new HashSet<String>(set2);
 +        set2.retainAll(userAuths);
 +
 +        if (set1.containsAll(set2) && expected.containsKey(set2)) {
 +          e.addAll(expected.get(set2));
 +        }
 +      }
 +
 +      set1.retainAll(userAuths);
 +      verify(c, tableName, set1, e);
 +    }
 +
 +  }
 +
 +  private void queryDefaultData(Connector c, String tableName) throws Exception {
 +    Scanner scanner;
 +
 +    // should return no records
 +    c.securityOperations().changeUserAuthorizations("root", new Authorizations("BASE", "DEFLABEL"));
 +    scanner = getConnector().createScanner(tableName, new Authorizations());
 +    verifyDefault(scanner, 0);
 +
 +    // should return one record
 +    scanner = getConnector().createScanner(tableName, new Authorizations("BASE"));
 +    verifyDefault(scanner, 1);
 +
 +    // should return all three records
 +    scanner = getConnector().createScanner(tableName, new Authorizations("BASE", "DEFLABEL"));
 +    verifyDefault(scanner, 3);
 +  }
 +
 +  private void verifyDefault(Scanner scanner, int expectedCount) throws Exception {
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner)
 +      --expectedCount;
 +    if (expectedCount != 0)
 +      throw new Exception(" expected count !=0 " + expectedCount);
 +  }
 +
 +  private void verify(Connector c, String tableName, Set<String> auths, Set<String> expectedValues) throws Exception {
 +    ByteArraySet bas = nbas(auths);
 +
 +    try {
 +      verify(c, tableName, bas, expectedValues.toArray(new String[0]));
 +    } catch (Exception e) {
 +      throw new Exception("Verification failed auths=" + auths + " exp=" + expectedValues, e);
 +    }
 +  }
 +
 +  private ByteArraySet nbas(Set<String> auths) {
 +    ByteArraySet bas = new ByteArraySet();
 +    for (String auth : auths) {
-       bas.add(auth.getBytes(Constants.UTF8));
++      bas.add(auth.getBytes(UTF_8));
 +    }
 +    return bas;
 +  }
 +
 +  private void verify(Connector c, String tableName, ByteArraySet nss, String... expected) throws Exception {
 +    Scanner scanner = c.createScanner(tableName, new Authorizations(nss));
 +    verify(scanner.iterator(), expected);
 +
 +    BatchScanner bs = getConnector().createBatchScanner(tableName, new Authorizations(nss), 3);
 +    bs.setRanges(Collections.singleton(new Range()));
 +    verify(bs.iterator(), expected);
 +    bs.close();
 +  }
 +
 +  private void verify(Iterator<Entry<Key,Value>> iter, String... expected) throws Exception {
 +    HashSet<String> valuesSeen = new HashSet<String>();
 +
 +    while (iter.hasNext()) {
 +      Entry<Key,Value> entry = iter.next();
 +      if (valuesSeen.contains(entry.getValue().toString())) {
 +        throw new Exception("Value seen twice");
 +      }
 +      valuesSeen.add(entry.getValue().toString());
 +    }
 +
 +    for (String ev : expected) {
 +      if (!valuesSeen.remove(ev)) {
 +        throw new Exception("Did not see expected value " + ev);
 +      }
 +    }
 +
 +    if (valuesSeen.size() != 0) {
 +      throw new Exception("Saw more values than expected " + valuesSeen);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------


[35/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
index d0e6aea,0000000..258080c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
@@@ -1,775 -1,0 +1,777 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.nio.ByteBuffer;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.SecurityOperationsImpl;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.TColumn;
 +import org.apache.accumulo.core.data.thrift.TKeyExtent;
 +import org.apache.accumulo.core.data.thrift.TRange;
 +import org.apache.accumulo.core.master.thrift.FateOperation;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.security.handler.Authenticator;
 +import org.apache.accumulo.server.security.handler.Authorizor;
 +import org.apache.accumulo.server.security.handler.PermissionHandler;
 +import org.apache.accumulo.server.security.handler.ZKAuthenticator;
 +import org.apache.accumulo.server.security.handler.ZKAuthorizor;
 +import org.apache.accumulo.server.security.handler.ZKPermHandler;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * Utility class for performing various security operations with the appropriate checks
 + */
 +public class SecurityOperation {
 +  private static final Logger log = Logger.getLogger(SecurityOperationsImpl.class);
 +
 +  protected Authorizor authorizor;
 +  protected Authenticator authenticator;
 +  protected PermissionHandler permHandle;
 +  private static String rootUserName = null;
 +  private final ZooCache zooCache;
 +  private final String ZKUserPath;
 +
 +  static SecurityOperation instance;
 +
 +  public static synchronized SecurityOperation getInstance() {
 +    String instanceId = HdfsZooInstance.getInstance().getInstanceID();
 +    return getInstance(instanceId, false);
 +  }
 +
 +  public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) {
 +    if (instance == null) {
 +      instance = new SecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, initialize),
 +          instanceId);
 +    }
 +    return instance;
 +  }
 +
 +  protected static Authorizor getAuthorizor(String instanceId, boolean initialize) {
 +    Authorizor toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHORIZOR, Authorizor.class,
 +        ZKAuthorizor.getInstance());
 +    toRet.initialize(instanceId, initialize);
 +    return toRet;
 +  }
 +
 +  protected static Authenticator getAuthenticator(String instanceId, boolean initialize) {
 +    Authenticator toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHENTICATOR, Authenticator.class,
 +        ZKAuthenticator.getInstance());
 +    toRet.initialize(instanceId, initialize);
 +    return toRet;
 +  }
 +
 +  protected static PermissionHandler getPermHandler(String instanceId, boolean initialize) {
 +    PermissionHandler toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_PERMISSION_HANDLER,
 +        PermissionHandler.class, ZKPermHandler.getInstance());
 +    toRet.initialize(instanceId, initialize);
 +    return toRet;
 +  }
 +
 +  protected SecurityOperation(String instanceId) {
 +    ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
 +    zooCache = new ZooCache();
 +  }
 +
 +  public SecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) {
 +    this(instanceId);
 +    authorizor = author;
 +    authenticator = authent;
 +    permHandle = pm;
 +
 +    if (!authorizor.validSecurityHandlers(authenticator, pm) || !authenticator.validSecurityHandlers(authorizor, pm)
 +        || !permHandle.validSecurityHandlers(authent, author))
 +      throw new RuntimeException(authorizor + ", " + authenticator + ", and " + pm
 +          + " do not play nice with eachother. Please choose authentication and authorization mechanisms that are compatible with one another.");
 +  }
 +
 +  public void initializeSecurity(TCredentials credentials, String rootPrincipal, byte[] token) throws AccumuloSecurityException, ThriftSecurityException {
 +    authenticate(credentials);
 +
 +    if (!isSystemUser(credentials))
 +      throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    authenticator.initializeSecurity(credentials, rootPrincipal, token);
 +    authorizor.initializeSecurity(credentials, rootPrincipal);
 +    permHandle.initializeSecurity(credentials, rootPrincipal);
 +    try {
 +      permHandle.grantTablePermission(rootPrincipal, MetadataTable.ID, TablePermission.ALTER_TABLE);
 +    } catch (TableNotFoundException e) {
 +      // Shouldn't happen
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  public synchronized String getRootUsername() {
 +    if (rootUserName == null)
-       rootUserName = new String(zooCache.get(ZKUserPath), Constants.UTF8);
++      rootUserName = new String(zooCache.get(ZKUserPath), UTF_8);
 +    return rootUserName;
 +  }
 +
 +  public boolean isSystemUser(TCredentials credentials) {
 +    return SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName());
 +  }
 +
 +  protected void authenticate(TCredentials credentials) throws ThriftSecurityException {
 +    if (!credentials.getInstanceId().equals(HdfsZooInstance.getInstance().getInstanceID()))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.INVALID_INSTANCEID);
 +
 +    Credentials creds = Credentials.fromThrift(credentials);
 +    if (isSystemUser(credentials)) {
 +      if (!(SystemCredentials.get().equals(creds))) {
 +        throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
 +      }
 +    } else {
 +      try {
 +        if (!authenticator.authenticateUser(creds.getPrincipal(), creds.getToken())) {
 +          throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
 +        }
 +      } catch (AccumuloSecurityException e) {
 +        log.debug(e);
 +        throw e.asThriftException();
 +      }
 +    }
 +  }
 +
 +  public boolean canAskAboutUser(TCredentials credentials, String user) throws ThriftSecurityException {
 +    // Authentication done in canPerformSystemActions
 +    if (!(canPerformSystemActions(credentials) || credentials.getPrincipal().equals(user)))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return true;
 +  }
 +
 +  public boolean authenticateUser(TCredentials credentials, TCredentials toAuth) throws ThriftSecurityException {
 +    canAskAboutUser(credentials, toAuth.getPrincipal());
 +    // User is already authenticated from canAskAboutUser
 +    if (credentials.equals(toAuth))
 +      return true;
 +    try {
 +      Credentials toCreds = Credentials.fromThrift(toAuth);
 +      return authenticator.authenticateUser(toCreds.getPrincipal(), toCreds.getToken());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public Authorizations getUserAuthorizations(TCredentials credentials, String user) throws ThriftSecurityException {
 +    authenticate(credentials);
 +
 +    targetUserExists(user);
 +
 +    if (!credentials.getPrincipal().equals(user) && !hasSystemPermission(credentials, SystemPermission.SYSTEM, false))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    try {
 +      return authorizor.getCachedUserAuthorizations(user);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public Authorizations getUserAuthorizations(TCredentials credentials) throws ThriftSecurityException {
 +    // system user doesn't need record-level authorizations for the tables it reads
 +    if (isSystemUser(credentials)) {
 +      authenticate(credentials);
 +      return Authorizations.EMPTY;
 +    }
 +    return getUserAuthorizations(credentials, credentials.getPrincipal());
 +  }
 +
 +  public boolean userHasAuthorizations(TCredentials credentials, List<ByteBuffer> list) throws ThriftSecurityException {
 +    authenticate(credentials);
 +
 +    if (isSystemUser(credentials)) {
 +      // system user doesn't need record-level authorizations for the tables it reads (for now)
 +      return list.isEmpty();
 +    }
 +
 +    try {
 +      return authorizor.isValidAuthorizations(credentials.getPrincipal(), list);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  private boolean hasSystemPermission(TCredentials credentials, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
 +    return hasSystemPermissionWithNamespaceId(credentials, permission, null, useCached);
 +  }
 +
 +  /**
 +   * Checks if a user has a system permission
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  private boolean hasSystemPermissionWithNamespaceId(TCredentials credentials, SystemPermission permission, String namespaceId, boolean useCached)
 +      throws ThriftSecurityException {
 +    if (isSystemUser(credentials))
 +      return true;
 +
 +    if (_hasSystemPermission(credentials.getPrincipal(), permission, useCached))
 +      return true;
 +    if (namespaceId != null) {
 +      return _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
 +    }
 +
 +    return false;
 +  }
 +
 +  /**
 +   * Checks if a user has a system permission<br/>
 +   * This cannot check if a system user has permission.
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  private boolean _hasSystemPermission(String user, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
 +    if (user.equals(getRootUsername()))
 +      return true;
 +
 +    targetUserExists(user);
 +
 +    try {
 +      if (useCached)
 +        return permHandle.hasCachedSystemPermission(user, permission);
 +      return permHandle.hasSystemPermission(user, permission);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  /**
 +   * Checks if a user has a table permission
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  protected boolean hasTablePermission(TCredentials credentials, String tableId, String namespaceId, TablePermission permission, boolean useCached) throws ThriftSecurityException {
 +    if (isSystemUser(credentials))
 +      return true;
 +    return _hasTablePermission(credentials.getPrincipal(), tableId, permission, useCached)
 +        || _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
 +  }
 +
 +  /**
 +   * Checks if a user has a table permission<br/>
 +   * This cannot check if a system user has permission.
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  protected boolean _hasTablePermission(String user, String table, TablePermission permission, boolean useCached) throws ThriftSecurityException {
 +    targetUserExists(user);
 +
 +    if ((table.equals(MetadataTable.ID) || table.equals(RootTable.ID)) && permission.equals(TablePermission.READ))
 +      return true;
 +
 +    try {
 +      if (useCached)
 +        return permHandle.hasCachedTablePermission(user, table, permission);
 +      return permHandle.hasTablePermission(user, table, permission);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(user, SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  /**
 +   * Checks if a user has a namespace permission<br/>
 +   * This cannot check if a system user has permission.
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  protected boolean _hasNamespacePermission(String user, String namespace, NamespacePermission permission, boolean useCached) throws ThriftSecurityException {
 +    if (permission == null)
 +      return false;
 +
 +    targetUserExists(user);
 +
 +    if (namespace.equals(Namespaces.ACCUMULO_NAMESPACE_ID) && permission.equals(NamespacePermission.READ))
 +      return true;
 +
 +    try {
 +      if (useCached)
 +        return permHandle.hasCachedNamespacePermission(user, namespace, permission);
 +      return permHandle.hasNamespacePermission(user, namespace, permission);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(user, SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  // some people just aren't allowed to ask about other users; here are those who can ask
 +  private boolean canAskAboutOtherUsers(TCredentials credentials, String user) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return credentials.getPrincipal().equals(user) || hasSystemPermission(credentials, SystemPermission.SYSTEM, false)
 +        || hasSystemPermission(credentials, SystemPermission.CREATE_USER, false) || hasSystemPermission(credentials, SystemPermission.ALTER_USER, false)
 +        || hasSystemPermission(credentials, SystemPermission.DROP_USER, false);
 +  }
 +
 +  private void targetUserExists(String user) throws ThriftSecurityException {
 +    if (user.equals(getRootUsername()))
 +      return;
 +    try {
 +      if (!authenticator.userExists(user))
 +        throw new ThriftSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public boolean canScan(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, true);
 +  }
 +
 +  public boolean canScan(TCredentials credentials, String tableId, String namespaceId, TRange range, List<TColumn> columns, List<IterInfo> ssiList,
 +      Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
 +    return canScan(credentials, tableId, namespaceId);
 +  }
 +
 +  public boolean canScan(TCredentials credentials, String table, String namespaceId, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList,
 +      Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
 +    return canScan(credentials, table, namespaceId);
 +  }
 +
 +  public boolean canWrite(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasTablePermission(credentials, tableId, namespaceId, TablePermission.WRITE, true);
 +  }
 +
 +  public boolean canConditionallyUpdate(TCredentials credentials, String tableID, String namespaceId, List<ByteBuffer> authorizations) throws ThriftSecurityException {
 +
 +    authenticate(credentials);
 +
 +    return hasTablePermission(credentials, tableID, namespaceId, TablePermission.WRITE, true) && hasTablePermission(credentials, tableID, namespaceId, TablePermission.READ, true);
 +  }
 +
 +  public boolean canSplitTablet(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasSystemPermissionWithNamespaceId(credentials, SystemPermission.SYSTEM, namespaceId, false)
 +        || hasTablePermission(credentials, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  /**
 +   * This is the check to perform any system action. This includes tserver's loading of a tablet, shutting the system down, or altering system properties.
 +   */
 +  public boolean canPerformSystemActions(TCredentials credentials) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermission(credentials, SystemPermission.SYSTEM, false);
 +  }
 +
 +  public boolean canFlush(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canAlterTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false)
 +        || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false);
 +  }
 +
 +  public boolean canCreateTable(TCredentials c, String table, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, namespaceId, false);
 +  }
 +
 +  public boolean canRenameTable(TCredentials c, String tableId, String oldTableName, String newTableName, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canCloneTable(TCredentials c, String tableId, String tableName, String destinationNamespaceId, String srcNamespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, destinationNamespaceId, false) && hasTablePermission(c, tableId, srcNamespaceId, TablePermission.READ, false);
 +  }
 +
 +  public boolean canDeleteTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.DROP_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.DROP_TABLE, false);
 +  }
 +
 +  public boolean canOnlineOfflineTable(TCredentials c, String tableId, FateOperation op, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
 +        || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canMerge(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
 +        || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canDeleteRange(TCredentials c, String tableId, String tableName, Text startRow, Text endRow, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
 +  }
 +
 +  public boolean canBulkImport(TCredentials c, String tableId, String tableName, String dir, String failDir, String namespaceId) throws ThriftSecurityException {
 +    return canBulkImport(c, tableId, namespaceId);
 +  }
 +
 +  public boolean canBulkImport(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasTablePermission(c, tableId, namespaceId, TablePermission.BULK_IMPORT, false);
 +  }
 +
 +  public boolean canCompact(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
 +  }
 +
 +  public boolean canChangeAuthorizations(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermission(c, SystemPermission.ALTER_USER, false);
 +  }
 +
 +  public boolean canChangePassword(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    return c.getPrincipal().equals(user) || hasSystemPermission(c, SystemPermission.ALTER_USER, false);
 +  }
 +
 +  public boolean canCreateUser(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermission(c, SystemPermission.CREATE_USER, false);
 +  }
 +
 +  public boolean canDropUser(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    if (user.equals(getRootUsername()))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return hasSystemPermission(c, SystemPermission.DROP_USER, false);
 +  }
 +
 +  public boolean canGrantSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
 +    authenticate(c);
 +    // can't grant GRANT
 +    if (sysPerm.equals(SystemPermission.GRANT))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
 +    return hasSystemPermission(c, SystemPermission.GRANT, false);
 +  }
 +
 +  public boolean canGrantTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
 +  }
 +
 +  public boolean canGrantNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
 +    return canModifyNamespacePermission(c, user, namespace);
 +  }
 +
 +  private boolean canModifyNamespacePermission(TCredentials c, String user, String namespace) throws ThriftSecurityException {
 +    authenticate(c);
 +    // The one case where Table/SystemPermission -> NamespacePermission breaks down. The alternative is to make SystemPermission.ALTER_NAMESPACE provide
 +    // NamespacePermission.GRANT & ALTER_NAMESPACE, but then it would cause some permission checks to succeed with GRANT when they shouldn't
 +    
 +    // This is a bit hackier then I (vines) wanted, but I think this one hackiness makes the overall SecurityOperations more succinct.
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_NAMESPACE, namespace, false)
 +        || hasNamespacePermission(c, c.principal, namespace, NamespacePermission.GRANT);
 +  }
 +
 +  public boolean canRevokeSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
 +    authenticate(c);
 +    // can't modify root user
 +    if (user.equals(getRootUsername()))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    // can't revoke GRANT
 +    if (sysPerm.equals(SystemPermission.GRANT))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
 +
 +    return hasSystemPermission(c, SystemPermission.GRANT, false);
 +  }
 +
 +  public boolean canRevokeTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
 +  }
 +
 +  public boolean canRevokeNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
 +    return canModifyNamespacePermission(c, user, namespace);
 +  }
 +
 +  public void changeAuthorizations(TCredentials credentials, String user, Authorizations authorizations) throws ThriftSecurityException {
 +    if (!canChangeAuthorizations(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      authorizor.changeAuthorizations(user, authorizations);
 +      log.info("Changed authorizations for user " + user + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException ase) {
 +      throw ase.asThriftException();
 +    }
 +  }
 +
 +  public void changePassword(TCredentials credentials, Credentials toChange) throws ThriftSecurityException {
 +    if (!canChangePassword(credentials, toChange.getPrincipal()))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      AuthenticationToken token = toChange.getToken();
 +      authenticator.changePassword(toChange.getPrincipal(), token);
 +      log.info("Changed password for user " + toChange.getPrincipal() + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void createUser(TCredentials credentials, Credentials newUser, Authorizations authorizations) throws ThriftSecurityException {
 +    if (!canCreateUser(credentials, newUser.getPrincipal()))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      AuthenticationToken token = newUser.getToken();
 +      authenticator.createUser(newUser.getPrincipal(), token);
 +      authorizor.initUser(newUser.getPrincipal());
 +      permHandle.initUser(newUser.getPrincipal());
 +      log.info("Created user " + newUser.getPrincipal() + " at the request of user " + credentials.getPrincipal());
 +      if (canChangeAuthorizations(credentials, newUser.getPrincipal()))
 +        authorizor.changeAuthorizations(newUser.getPrincipal(), authorizations);
 +    } catch (AccumuloSecurityException ase) {
 +      throw ase.asThriftException();
 +    }
 +  }
 +
 +  public void dropUser(TCredentials credentials, String user) throws ThriftSecurityException {
 +    if (!canDropUser(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      authorizor.dropUser(user);
 +      authenticator.dropUser(user);
 +      permHandle.cleanUser(user);
 +      log.info("Deleted user " + user + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void grantSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
 +    if (!canGrantSystem(credentials, user, permissionById))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.grantSystemPermission(user, permissionById);
 +      log.info("Granted system permission " + permissionById + " for user " + user + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void grantTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
 +    if (!canGrantTable(c, user, tableId, namespaceId))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.grantTablePermission(user, tableId, permission);
 +      log.info("Granted table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void grantNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
 +    if (!canGrantNamespace(c, user, namespace))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.grantNamespacePermission(user, namespace, permission);
 +      log.info("Granted namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
 +          + c.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void revokeSystemPermission(TCredentials credentials, String user, SystemPermission permission) throws ThriftSecurityException {
 +    if (!canRevokeSystem(credentials, user, permission))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.revokeSystemPermission(user, permission);
 +      log.info("Revoked system permission " + permission + " for user " + user + " at the request of user " + credentials.getPrincipal());
 +
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void revokeTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
 +    if (!canRevokeTable(c, user, tableId, namespaceId))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.revokeTablePermission(user, tableId, permission);
 +      log.info("Revoked table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
 +
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void revokeNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
 +    if (!canRevokeNamespace(c, user, namespace))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.revokeNamespacePermission(user, namespace, permission);
 +      log.info("Revoked namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
 +          + c.getPrincipal());
 +
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public boolean hasSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
 +    if (!canAskAboutOtherUsers(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return _hasSystemPermission(user, permissionById, false);
 +  }
 +
 +  public boolean hasTablePermission(TCredentials credentials, String user, String tableId, TablePermission permissionById) throws ThriftSecurityException {
 +    if (!canAskAboutOtherUsers(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return _hasTablePermission(user, tableId, permissionById, false);
 +  }
 +
 +  public boolean hasNamespacePermission(TCredentials credentials, String user, String namespace, NamespacePermission permissionById)
 +      throws ThriftSecurityException {
 +    if (!canAskAboutOtherUsers(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return _hasNamespacePermission(user, namespace, permissionById, false);
 +  }
 +
 +  public Set<String> listUsers(TCredentials credentials) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    try {
 +      return authenticator.listUsers();
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void deleteTable(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    if (!canDeleteTable(credentials, tableId, namespaceId))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      permHandle.cleanTablePermissions(tableId);
 +    } catch (AccumuloSecurityException e) {
 +      e.setUser(credentials.getPrincipal());
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void deleteNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
 +    if (!canDeleteNamespace(credentials, namespace))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      permHandle.cleanNamespacePermissions(namespace);
 +    } catch (AccumuloSecurityException e) {
 +      e.setUser(credentials.getPrincipal());
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public boolean canExport(TCredentials credentials, String tableId, String tableName, String exportDir, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, false);
 +  }
 +
 +  public boolean canImport(TCredentials credentials, String tableName, String importDir, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.CREATE_TABLE, namespaceId, false);
 +  }
 +
 +  public boolean canAlterNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
 +  }
 +
 +  public boolean canCreateNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
 +    return canCreateNamespace(credentials);
 +  }
 +
 +  private boolean canCreateNamespace(TCredentials credentials) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermission(credentials, SystemPermission.CREATE_NAMESPACE, false);
 +  }
 +
 +  public boolean canDeleteNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.DROP_NAMESPACE, namespaceId, false);
 +  }
 +
 +  public boolean canRenameNamespace(TCredentials credentials, String namespaceId, String oldName, String newName) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 19e7ff7,0000000..872fe11
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@@ -1,149 -1,0 +1,151 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.security.MessageDigest;
 +import java.security.NoSuchAlgorithmException;
 +import java.security.SecurityPermission;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.hadoop.io.Writable;
 +
 +/**
 + * Credentials for the system services.
 + * 
 + * @since 1.6.0
 + */
 +public final class SystemCredentials extends Credentials {
 +
 +  private static final SecurityPermission SYSTEM_CREDENTIALS_PERMISSION = new SecurityPermission("systemCredentialsPermission");
 +
 +  private static SystemCredentials SYSTEM_CREDS = null;
 +  private static final String SYSTEM_PRINCIPAL = "!SYSTEM";
 +
 +  private final TCredentials AS_THRIFT;
 +
 +  SystemCredentials(Instance instance) {
 +    super(SYSTEM_PRINCIPAL, SystemToken.get(instance));
 +    AS_THRIFT = super.toThrift(instance);
 +  }
 +
 +  public static SystemCredentials get() {
 +    check_permission();
 +    if (SYSTEM_CREDS == null) {
 +      SYSTEM_CREDS = new SystemCredentials(HdfsZooInstance.getInstance());
 +    }
 +    return SYSTEM_CREDS;
 +  }
 +
 +  private static void check_permission() {
 +    SecurityManager sm = System.getSecurityManager();
 +    if (sm != null) {
 +      sm.checkPermission(SYSTEM_CREDENTIALS_PERMISSION);
 +    }
 +  }
 +
 +  public static SystemCredentials get(Instance instance) {
 +    check_permission();
 +    /* Special case to avoid duplicating SYSTEM_CREDS */
 +    if (null != SYSTEM_CREDS) {
 +      if (SYSTEM_CREDS.AS_THRIFT.getInstanceId().equals(instance.getInstanceID())) {
 +        return SYSTEM_CREDS;
 +      }
 +    }
 +    return new SystemCredentials(instance);
 +  }
 +
 +  @Override
 +  public TCredentials toThrift(Instance instance) {
 +    if (!AS_THRIFT.getInstanceId().equals(instance.getInstanceID()))
 +      throw new IllegalArgumentException("Unexpected instance used for " + SystemCredentials.class.getSimpleName() + ": " + instance.getInstanceID());
 +    return AS_THRIFT;
 +  }
 +
 +  /**
 +   * An {@link AuthenticationToken} type for Accumulo servers for inter-server communication.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static final class SystemToken extends PasswordToken {
 +
 +    /**
 +     * A Constructor for {@link Writable}.
 +     */
 +    public SystemToken() {}
 +
 +    private SystemToken(byte[] systemPassword) {
 +      super(systemPassword);
 +    }
 +
 +    private static SystemToken get(Instance instance) {
-       byte[] instanceIdBytes = instance.getInstanceID().getBytes(Constants.UTF8);
++      byte[] instanceIdBytes = instance.getInstanceID().getBytes(UTF_8);
 +      byte[] confChecksum;
 +      MessageDigest md;
 +      try {
 +        md = MessageDigest.getInstance(Constants.PW_HASH_ALGORITHM);
 +      } catch (NoSuchAlgorithmException e) {
 +        throw new RuntimeException("Failed to compute configuration checksum", e);
 +      }
 +
 +      // seed the config with the version and instance id, so at least it's not empty
-       md.update(ServerConstants.WIRE_VERSION.toString().getBytes(Constants.UTF8));
++      md.update(ServerConstants.WIRE_VERSION.toString().getBytes(UTF_8));
 +      md.update(instanceIdBytes);
 +
 +      for (Entry<String,String> entry : ServerConfiguration.getSiteConfiguration()) {
 +        // only include instance properties
 +        if (entry.getKey().startsWith(Property.INSTANCE_PREFIX.toString())) {
-           md.update(entry.getKey().getBytes(Constants.UTF8));
-           md.update(entry.getValue().getBytes(Constants.UTF8));
++          md.update(entry.getKey().getBytes(UTF_8));
++          md.update(entry.getValue().getBytes(UTF_8));
 +        }
 +      }
 +      confChecksum = md.digest();
 +
 +      int wireVersion = ServerConstants.WIRE_VERSION;
 +
 +      ByteArrayOutputStream bytes = new ByteArrayOutputStream(3 * (Integer.SIZE / Byte.SIZE) + instanceIdBytes.length + confChecksum.length);
 +      DataOutputStream out = new DataOutputStream(bytes);
 +      try {
 +        out.write(wireVersion * -1);
 +        out.write(instanceIdBytes.length);
 +        out.write(instanceIdBytes);
 +        out.write(confChecksum.length);
 +        out.write(confChecksum);
 +      } catch (IOException e) {
 +        // this is impossible with ByteArrayOutputStream; crash hard if this happens
 +        throw new RuntimeException(e);
 +      }
 +      return new SystemToken(Base64.encodeBase64(bytes.toByteArray()));
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
index 5f6ff71,0000000..fa1632c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
@@@ -1,214 -1,0 +1,216 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +// Utility class for adding all authentication info into ZK
 +public final class ZKAuthenticator implements Authenticator {
 +  static final Logger log = Logger.getLogger(ZKAuthenticator.class);
 +  private static Authenticator zkAuthenticatorInstance = null;
 +
 +  private String ZKUserPath;
 +  private final ZooCache zooCache;
 +
 +  public static synchronized Authenticator getInstance() {
 +    if (zkAuthenticatorInstance == null)
 +      zkAuthenticatorInstance = new ZKAuthenticator();
 +    return zkAuthenticatorInstance;
 +  }
 +
 +  public ZKAuthenticator() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException {
 +    try {
 +      // remove old settings from zookeeper first, if any
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        if (zoo.exists(ZKUserPath)) {
 +          zoo.recursiveDelete(ZKUserPath, NodeMissingPolicy.SKIP);
 +          log.info("Removed " + ZKUserPath + "/" + " from zookeeper");
 +        }
 +
 +        // prep parent node of users with root username
-         zoo.putPersistentData(ZKUserPath, principal.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++        zoo.putPersistentData(ZKUserPath, principal.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +        constructUser(principal, ZKSecurityTool.createPass(token));
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (AccumuloException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets up the user in ZK for the provided user. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void constructUser(String user, byte[] pass) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      zoo.putPrivatePersistentData(ZKUserPath + "/" + user, pass, NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  @Override
 +  public Set<String> listUsers() {
 +    return new TreeSet<String>(zooCache.getChildren(ZKUserPath));
 +  }
 +
 +  /**
 +   * Creates a user with no permissions whatsoever
 +   */
 +  @Override
 +  public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
 +    try {
 +      if (!(token instanceof PasswordToken))
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
 +      PasswordToken pt = (PasswordToken) token;
 +      constructUser(principal, ZKSecurityTool.createPass(pt.getPassword()));
 +    } catch (KeeperException e) {
 +      if (e.code().equals(KeeperException.Code.NODEEXISTS))
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_EXISTS, e);
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (AccumuloException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
 +    }
 +  }
 +
 +  @Override
 +  public void dropUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        ZooReaderWriter.getInstance().recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    }
 +  }
 +
 +  @Override
 +  public void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException {
 +    if (!(token instanceof PasswordToken))
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
 +    PasswordToken pt = (PasswordToken) token;
 +    if (userExists(principal)) {
 +      try {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + principal);
 +          ZooReaderWriter.getInstance().putPrivatePersistentData(ZKUserPath + "/" + principal, ZKSecurityTool.createPass(pt.getPassword()),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      } catch (KeeperException e) {
 +        log.error(e, e);
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
 +      } catch (InterruptedException e) {
 +        log.error(e, e);
 +        throw new RuntimeException(e);
 +      } catch (AccumuloException e) {
 +        log.error(e, e);
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
 +      }
 +    } else
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
 +  }
 +
 +  /**
 +   * Checks if a user exists
 +   */
 +  @Override
 +  public boolean userExists(String user) {
 +    return zooCache.get(ZKUserPath + "/" + user) != null;
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm) {
 +    return true;
 +  }
 +
 +  @Override
 +  public boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
 +    if (!(token instanceof PasswordToken))
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
 +    PasswordToken pt = (PasswordToken) token;
 +    byte[] pass;
 +    String zpath = ZKUserPath + "/" + principal;
 +    pass = zooCache.get(zpath);
 +    boolean result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
 +    if (!result) {
 +      zooCache.clear(zpath);
 +      pass = zooCache.get(zpath);
 +      result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
 +    }
 +    return result;
 +  }
 +
 +  @Override
 +  public Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes() {
 +    Set<Class<? extends AuthenticationToken>> cs = new HashSet<Class<? extends AuthenticationToken>>();
 +    cs.add(PasswordToken.class);
 +    return cs;
 +  }
 +
 +  @Override
 +  public boolean validTokenClass(String tokenClass) {
 +    return tokenClass.equals(PasswordToken.class.getName());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 75b73fc,0000000..78597ad
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,171 -1,0 +1,172 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.nio.ByteBuffer;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class ZKAuthorizor implements Authorizor {
 +  private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
 +  private static Authorizor zkAuthorizorInstance = null;
 +
 +  private final String ZKUserAuths = "/Authorizations";
 +
 +  private String ZKUserPath;
 +  private final ZooCache zooCache;
 +
 +  public static synchronized Authorizor getInstance() {
 +    if (zkAuthorizorInstance == null)
 +      zkAuthorizorInstance = new ZKAuthorizor();
 +    return zkAuthorizorInstance;
 +  }
 +
 +  public ZKAuthorizor() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
 +  }
 +
 +  @Override
 +  public Authorizations getCachedUserAuthorizations(String user) {
 +    byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
 +    if (authsBytes != null)
 +      return ZKSecurityTool.convertAuthorizations(authsBytes);
 +    return Authorizations.EMPTY;
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
 +    return true;
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    // create the root user with all system privileges, no table privileges, and no record-level authorizations
 +    Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
 +    for (SystemPermission p : SystemPermission.values())
 +      rootPerms.add(p);
 +    Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
 +    // Allow the root user to flush the metadata tables
 +    tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +
 +    try {
 +      // prep parent node of users with root username
 +      if (!zoo.exists(ZKUserPath))
-         zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++        zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +      initUser(rootuser);
 +      zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initUser(String user) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void dropUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
 +        zooCache.clear(ZKUserPath + "/" + user);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +
 +    }
 +  }
 +
 +  @Override
 +  public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
 +            NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
 +    Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
 +    for (ByteBuffer auth : auths)
 +      if (!userauths.contains(auth))
 +        return false;
 +    return true;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index be4806b,0000000..41f880d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,515 -1,0 +1,516 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.Code;
 +
 +/**
 + *
 + */
 +public class ZKPermHandler implements PermissionHandler {
 +  private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
 +  private static PermissionHandler zkPermHandlerInstance = null;
 +
 +  private String ZKUserPath;
 +  private String ZKTablePath;
 +  private String ZKNamespacePath;
 +  private final ZooCache zooCache;
 +  private final String ZKUserSysPerms = "/System";
 +  private final String ZKUserTablePerms = "/Tables";
 +  private final String ZKUserNamespacePerms = "/Namespaces";
 +
 +  public static synchronized PermissionHandler getInstance() {
 +    if (zkPermHandlerInstance == null)
 +      zkPermHandlerInstance = new ZKPermHandler();
 +    return zkPermHandlerInstance;
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
 +    ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
 +    ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
 +  }
 +
 +  public ZKPermHandler() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
 +    byte[] serializedPerms;
 +    final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
 +      zrw.sync(path);
 +      serializedPerms = zrw.getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        // maybe the table was just deleted?
 +        try {
 +          // check for existence:
 +          zrw.getData(ZKTablePath + "/" + table, null);
 +          // it's there, you don't have permission
 +          return false;
 +        } catch (InterruptedException ex) {
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +          return false;
 +        } catch (KeeperException ex) {
 +          // not there, throw an informative exception
 +          if (e.code() == Code.NONODE) {
 +            throw new TableNotFoundException(null, table, "while checking permissions");
 +          }
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +        }
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
 +    byte[] serializedPerms;
 +    final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
 +      zrw.sync(path);
 +      serializedPerms = zrw.getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        // maybe the namespace was just deleted?
 +        try {
 +          // check for existence:
 +          zrw.getData(ZKNamespacePath + "/" + namespace, null);
 +          // it's there, you don't have permission
 +          return false;
 +        } catch (InterruptedException ex) {
 +          log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
 +          return false;
 +        } catch (KeeperException ex) {
 +          // not there, throw an informative exception
 +          if (e.code() == Code.NONODE) {
 +            throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
 +          }
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +        }
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
 +      NamespaceNotFoundException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    try {
 +      byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +      Set<SystemPermission> perms;
 +      if (permBytes == null) {
 +        perms = new TreeSet<SystemPermission>();
 +      } else {
 +        perms = ZKSecurityTool.convertSystemPermissions(permBytes);
 +      }
 +
 +      if (perms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear();
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
 +    Set<TablePermission> tablePerms;
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +    if (serializedPerms != null)
 +      tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
 +    else
 +      tablePerms = new TreeSet<TablePermission>();
 +
 +    try {
 +      if (tablePerms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
 +              ZKSecurityTool.convertTablePermissions(tablePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
 +    Set<NamespacePermission> namespacePerms;
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +    if (serializedPerms != null)
 +      namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
 +    else
 +      namespacePerms = new TreeSet<NamespacePermission>();
 +
 +    try {
 +      if (namespacePerms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
 +              ZKSecurityTool.convertNamespacePermissions(namespacePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +
 +    // User had no system permission, nothing to revoke.
 +    if (sysPermBytes == null)
 +      return;
 +
 +    Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
 +
 +    try {
 +      if (sysPerms.remove(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear();
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +
 +    // User had no table permission, nothing to revoke.
 +    if (serializedPerms == null)
 +      return;
 +
 +    Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
 +    try {
 +      if (tablePerms.remove(permission)) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        if (tablePerms.size() == 0)
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
 +        else
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +
 +    // User had no namespace permission, nothing to revoke.
 +    if (serializedPerms == null)
 +      return;
 +
 +    Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
 +    try {
 +      if (namespacePerms.remove(permission)) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        if (namespacePerms.size() == 0)
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
 +        else
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanTablePermissions(String table) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        for (String user : zooCache.getChildren(ZKUserPath))
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        for (String user : zooCache.getChildren(ZKUserPath))
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    // create the root user with all system privileges, no table privileges, and no record-level authorizations
 +    Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
 +    for (SystemPermission p : SystemPermission.values())
 +      rootPerms.add(p);
 +    Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
 +    // Allow the root user to flush the system tables
 +    tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
 +    Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
 +    namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
 +    namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
 +
 +    try {
 +      // prep parent node of users with root username
 +      if (!zoo.exists(ZKUserPath))
-         zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++        zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +      initUser(rootuser);
 +      zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
 +      for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
 +        createTablePerm(rootuser, entry.getKey(), entry.getValue());
 +      for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
 +        createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initUser(String user) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
 +      zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
 +      zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
 +          ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  /**
 +   * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
 +          ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
 +        zooCache.clear(ZKUserPath + "/" + user);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +
 +    }
 +  }
 +
 +  @Override
 +  public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] perms;
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserSysPerms;
 +      ZooReaderWriter.getInstance().sync(path);
 +      perms = ZooReaderWriter.getInstance().getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +
 +    if (perms == null)
 +      return false;
 +    return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
 +  }
 +
 +  @Override
 +  public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +    if (perms == null)
 +      return false;
 +    return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
 +    return true;
 +  }
 +
 +  @Override
 +  public void initTable(String table) throws AccumuloSecurityException {
 +    // All proper housekeeping is done on delete and permission granting, no work needs to be done here
 +  }
 +}


[15/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index e8d5bbf,0000000..002389a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@@ -1,399 -1,0 +1,400 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP;
 +
 +import java.nio.ByteBuffer;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +import org.apache.zookeeper.KeeperException.NotEmptyException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class LiveTServerSet implements Watcher {
 +
 +  public interface Listener {
 +    void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added);
 +  }
 +
 +  private static final Logger log = Logger.getLogger(LiveTServerSet.class);
 +
 +  private final Listener cback;
 +  private final Instance instance;
 +  private final AccumuloConfiguration conf;
 +  private ZooCache zooCache;
 +
 +  public class TServerConnection {
 +    private final HostAndPort address;
 +
 +    public TServerConnection(HostAndPort addr) throws TException {
 +      address = addr;
 +    }
 +
 +    private String lockString(ZooLock mlock) {
 +      return mlock.getLockID().serialize(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK);
 +    }
 +
 +    public void assignTablet(ZooLock lock, KeyExtent extent) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.loadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void unloadTablet(ZooLock lock, KeyExtent extent, boolean save) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.unloadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift(), save);
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public TabletServerStatus getTableMap(boolean usePooledConnection) throws TException, ThriftSecurityException {
 +
 +      if (usePooledConnection == true)
 +        throw new UnsupportedOperationException();
 +
 +      TTransport transport = ThriftUtil.createTransport(address, conf);
 +
 +      try {
 +        TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
 +        return client.getTabletServerStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
 +      } finally {
 +        if (transport != null)
 +          transport.close();
 +      }
 +    }
 +
 +    public void halt(ZooLock lock) throws TException, ThriftSecurityException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.halt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void fastHalt(ZooLock lock) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.fastHalt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void flush(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.flush(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
 +            startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void chop(ZooLock lock, KeyExtent extent) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.chop(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void splitTablet(ZooLock lock, KeyExtent extent, Text splitPoint) throws TException, ThriftSecurityException, NotServingTabletException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.splitTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), extent.toThrift(),
 +            ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength()));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void flushTablet(ZooLock lock, KeyExtent extent) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.flushTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void compact(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.compact(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
 +            startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public boolean isActive(long tid) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        return client.isActive(Tracer.traceInfo(), tid);
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +  }
 +
 +  static class TServerInfo {
 +    TServerConnection connection;
 +    TServerInstance instance;
 +
 +    TServerInfo(TServerInstance instance, TServerConnection connection) {
 +      this.connection = connection;
 +      this.instance = instance;
 +    }
 +  };
 +
 +  // The set of active tservers with locks, indexed by their name in zookeeper
 +  private Map<String,TServerInfo> current = new HashMap<String,TServerInfo>();
 +  // as above, indexed by TServerInstance
 +  private Map<TServerInstance,TServerInfo> currentInstances = new HashMap<TServerInstance,TServerInfo>();
 +
 +  // The set of entries in zookeeper without locks, and the first time each was noticed
 +  private Map<String,Long> locklessServers = new HashMap<String,Long>();
 +
 +  public LiveTServerSet(Instance instance, AccumuloConfiguration conf, Listener cback) {
 +    this.cback = cback;
 +    this.instance = instance;
 +    this.conf = conf;
 +
 +  }
 +
 +  public synchronized ZooCache getZooCache() {
 +    if (zooCache == null)
 +      zooCache = new ZooCache(this);
 +    return zooCache;
 +  }
 +
 +  public synchronized void startListeningForTabletServerChanges() {
 +    scanServers();
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        scanServers();
 +      }
 +    }, 0, 5000);
 +  }
 +
 +  public synchronized void scanServers() {
 +    try {
 +      final Set<TServerInstance> updates = new HashSet<TServerInstance>();
 +      final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
 +
 +      final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +
 +      HashSet<String> all = new HashSet<String>(current.keySet());
 +      all.addAll(getZooCache().getChildren(path));
 +
 +      locklessServers.keySet().retainAll(all);
 +
 +      for (String zPath : all) {
 +        checkServer(updates, doomed, path, zPath);
 +      }
 +
 +      // log.debug("Current: " + current.keySet());
 +      if (!doomed.isEmpty() || !updates.isEmpty())
 +        this.cback.update(this, doomed, updates);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +
 +  private void deleteServerNode(String serverNode) throws InterruptedException, KeeperException {
 +    try {
 +      ZooReaderWriter.getInstance().delete(serverNode, -1);
 +    } catch (NotEmptyException ex) {
 +      // race condition: tserver created the lock after our last check; we'll see it at the next check
 +    } catch (NoNodeException nne) {
 +      // someone else deleted it
 +    }
 +  }
 +
 +  private synchronized void checkServer(final Set<TServerInstance> updates, final Set<TServerInstance> doomed, final String path, final String zPath)
 +      throws TException, InterruptedException, KeeperException {
 +
 +    TServerInfo info = current.get(zPath);
 +
 +    final String lockPath = path + "/" + zPath;
 +    Stat stat = new Stat();
 +    byte[] lockData = ZooLock.getLockData(getZooCache(), lockPath, stat);
 +
 +    if (lockData == null) {
 +      if (info != null) {
 +        doomed.add(info.instance);
 +        current.remove(zPath);
 +        currentInstances.remove(info.instance);
 +      }
 +
 +      Long firstSeen = locklessServers.get(zPath);
 +      if (firstSeen == null) {
 +        locklessServers.put(zPath, System.currentTimeMillis());
 +      } else if (System.currentTimeMillis() - firstSeen > 10 * 60 * 1000) {
 +        deleteServerNode(path + "/" + zPath);
 +        locklessServers.remove(zPath);
 +      }
 +    } else {
 +      locklessServers.remove(zPath);
-       ServerServices services = new ServerServices(new String(lockData, Constants.UTF8));
++      ServerServices services = new ServerServices(new String(lockData, UTF_8));
 +      HostAndPort client = services.getAddress(ServerServices.Service.TSERV_CLIENT);
 +      TServerInstance instance = new TServerInstance(client, stat.getEphemeralOwner());
 +
 +      if (info == null) {
 +        updates.add(instance);
 +        TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
 +        current.put(zPath, tServerInfo);
 +        currentInstances.put(instance, tServerInfo);
 +      } else if (!info.instance.equals(instance)) {
 +        doomed.add(info.instance);
 +        updates.add(instance);
 +        TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
 +        current.put(zPath, tServerInfo);
 +        currentInstances.put(info.instance, tServerInfo);
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void process(WatchedEvent event) {
 +
 +    // its important that these event are propagated by ZooCache, because this ensures when reading zoocache that is has already processed the event and cleared
 +    // relevant nodes before code below reads from zoocache
 +
 +    if (event.getPath() != null) {
 +      if (event.getPath().endsWith(Constants.ZTSERVERS)) {
 +        scanServers();
 +      } else if (event.getPath().contains(Constants.ZTSERVERS)) {
 +        int pos = event.getPath().lastIndexOf('/');
 +
 +        // do only if ZTSERVER is parent
 +        if (pos >= 0 && event.getPath().substring(0, pos).endsWith(Constants.ZTSERVERS)) {
 +
 +          String server = event.getPath().substring(pos + 1);
 +
 +          final Set<TServerInstance> updates = new HashSet<TServerInstance>();
 +          final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
 +
 +          final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +
 +          try {
 +            checkServer(updates, doomed, path, server);
 +            if (!doomed.isEmpty() || !updates.isEmpty())
 +              this.cback.update(this, doomed, updates);
 +          } catch (Exception ex) {
 +            log.error(ex, ex);
 +          }
 +        }
 +      }
 +    }
 +  }
 +
 +  public synchronized TServerConnection getConnection(TServerInstance server) {
 +    if (server == null)
 +      return null;
 +    TServerInfo tServerInfo = currentInstances.get(server);
 +    if (tServerInfo == null)
 +      return null;
 +    return tServerInfo.connection;
 +  }
 +
 +  public synchronized Set<TServerInstance> getCurrentServers() {
 +    return new HashSet<TServerInstance>(currentInstances.keySet());
 +  }
 +
 +  public synchronized int size() {
 +    return current.size();
 +  }
 +
 +  public synchronized TServerInstance find(String tabletServer) {
 +    HostAndPort addr = AddressUtil.parseAddress(tabletServer, false);
 +    for (Entry<String,TServerInfo> entry : current.entrySet()) {
 +      if (entry.getValue().instance.getLocation().equals(addr))
 +        return entry.getValue().instance;
 +    }
 +    return null;
 +  }
 +
 +  public synchronized void remove(TServerInstance server) {
 +    String zPath = null;
 +    for (Entry<String,TServerInfo> entry : current.entrySet()) {
 +      if (entry.getValue().instance.equals(server)) {
 +        zPath = entry.getKey();
 +        break;
 +      }
 +    }
 +    if (zPath == null)
 +      return;
 +    current.remove(zPath);
 +    currentInstances.remove(server);
 +
 +    log.info("Removing zookeeper lock for " + server);
 +    String fullpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + zPath;
 +    try {
 +      ZooReaderWriter.getInstance().recursiveDelete(fullpath, SKIP);
 +    } catch (Exception e) {
 +      String msg = "error removing tablet server lock";
 +      log.fatal(msg, e);
 +      Halt.halt(msg, -1);
 +    }
 +    getZooCache().clear(fullpath);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
index 2f657c4,0000000..568e6f6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
@@@ -1,90 -1,0 +1,91 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.DeadServer;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.data.Stat;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +public class DeadServerList {
 +  private static final Logger log = Logger.getLogger(DeadServerList.class);
 +  private final String path;
 +  
 +  public DeadServerList(String path) {
 +    this.path = path;
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.mkdirs(path);
 +    } catch (Exception ex) {
 +      log.error("Unable to make parent directories of " + path, ex);
 +    }
 +  }
 +  
 +  public List<DeadServer> getList() {
 +    List<DeadServer> result = new ArrayList<DeadServer>();
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      List<String> children = zoo.getChildren(path);
 +      if (children != null) {
 +        for (String child : children) {
 +          Stat stat = new Stat();
 +          byte[] data;
 +          try {
 +            data = zoo.getData(path + "/" + child, stat);
 +          } catch (NoNodeException nne) {
 +            // Another thread or process can delete child while this loop is running.
 +            // We ignore this error since it's harmless if we miss the deleted server
 +            // in the dead server list.
 +            continue;
 +          }
-           DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, Constants.UTF8));
++          DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, UTF_8));
 +          result.add(server);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +    return result;
 +  }
 +  
 +  public void delete(String server) {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.recursiveDelete(path + "/" + server, NodeMissingPolicy.SKIP);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +  
 +  public void post(String server, String cause) {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
-       zoo.putPersistentData(path + "/" + server, cause.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++      zoo.putPersistentData(path + "/" + server, cause.getBytes(UTF_8), NodeExistsPolicy.SKIP);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
index 400d0ac,0000000..b4ef519
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
@@@ -1,143 -1,0 +1,144 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.ObjectInputStream;
 +import java.io.ObjectOutputStream;
 +import java.io.Serializable;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.hadoop.io.Text;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * A tablet is assigned to a tablet server at the given address as long as it is alive and well. When the tablet server is restarted, the instance information
 + * it advertises will change. Therefore tablet assignments can be considered out-of-date if the tablet server instance information has been changed.
 + * 
 + */
 +public class TServerInstance implements Comparable<TServerInstance>, Serializable {
 +  
 +  private static final long serialVersionUID = 1L;
 +  
 +  // HostAndPort is not Serializable
 +  private transient HostAndPort location;
 +  private String session;
 +  private String cachedStringRepresentation;
 +  
 +  public TServerInstance(HostAndPort address, String session) {
 +    this.location = address;
 +    this.session = session;
 +    this.cachedStringRepresentation = hostPort() + "[" + session + "]";
 +  }
 +  
 +  public TServerInstance(HostAndPort address, long session) {
 +    this(address, Long.toHexString(session));
 +  }
 +  
 +  public TServerInstance(String address, long session) {
 +    this(AddressUtil.parseAddress(address, false), Long.toHexString(session));
 +  }
 +  
 +  public TServerInstance(Value address, Text session) {
-     this(AddressUtil.parseAddress(new String(address.get(), Constants.UTF8), false), session.toString());
++    this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString());
 +  }
 +  
 +  public void putLocation(Mutation m) {
 +    m.put(TabletsSection.CurrentLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
 +  }
 +  
 +  public void putFutureLocation(Mutation m) {
 +    m.put(TabletsSection.FutureLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
 +  }
 +  
 +  public void putLastLocation(Mutation m) {
 +    m.put(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
 +  }
 +  
 +  public void clearLastLocation(Mutation m) {
 +    m.putDelete(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier());
 +  }
 +  
 +  @Override
 +  public int compareTo(TServerInstance other) {
 +    if (this == other)
 +      return 0;
 +    return this.toString().compareTo(other.toString());
 +  }
 +  
 +  @Override
 +  public int hashCode() {
 +    return toString().hashCode();
 +  }
 +  
 +  @Override
 +  public boolean equals(Object obj) {
 +    if (obj instanceof TServerInstance) {
 +      return compareTo((TServerInstance) obj) == 0;
 +    }
 +    return false;
 +  }
 +  
 +  @Override
 +  public String toString() {
 +    return cachedStringRepresentation;
 +  }
 +  
 +  public int port() {
 +    return getLocation().getPort();
 +  }
 +  
 +  public String host() {
 +    return getLocation().getHostText();
 +  }
 +  
 +  public String hostPort() {
 +    return getLocation().toString();
 +  }
 +  
 +  public Text asColumnQualifier() {
 +    return new Text(this.getSession());
 +  }
 +  
 +  public Value asMutationValue() {
-     return new Value(getLocation().toString().getBytes(Constants.UTF8));
++    return new Value(getLocation().toString().getBytes(UTF_8));
 +  }
 +  
 +  public HostAndPort getLocation() {
 +    return location;
 +  }
 +  
 +  public String getSession() {
 +    return session;
 +  }
 +  
 +  private void writeObject(ObjectOutputStream out) throws IOException {
 +    out.defaultWriteObject();
 +    out.writeObject(location.toString());
 +  }
 +  
 +  private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
 +    in.defaultReadObject();
 +    location = HostAndPort.fromString(in.readObject().toString());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
index f4d5591,0000000..91be9c3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
@@@ -1,189 -1,0 +1,190 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorEnvironment;
 +import org.apache.accumulo.core.iterators.SkippingIterator;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
 +import org.apache.hadoop.io.DataInputBuffer;
 +import org.apache.hadoop.io.DataOutputBuffer;
 +import org.apache.hadoop.io.Text;
 +
 +public class TabletStateChangeIterator extends SkippingIterator {
 +  
 +  private static final String SERVERS_OPTION = "servers";
 +  private static final String TABLES_OPTION = "tables";
 +  private static final String MERGES_OPTION = "merges";
 +  // private static final Logger log = Logger.getLogger(TabletStateChangeIterator.class);
 +  
 +  Set<TServerInstance> current;
 +  Set<String> onlineTables;
 +  Map<Text,MergeInfo> merges;
 +  
 +  @Override
 +  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
 +    super.init(source, options, env);
 +    current = parseServers(options.get(SERVERS_OPTION));
 +    onlineTables = parseTables(options.get(TABLES_OPTION));
 +    merges = parseMerges(options.get(MERGES_OPTION));
 +  }
 +  
 +  private Set<String> parseTables(String tables) {
 +    if (tables == null)
 +      return null;
 +    Set<String> result = new HashSet<String>();
 +    for (String table : tables.split(","))
 +      result.add(table);
 +    return result;
 +  }
 +  
 +  private Set<TServerInstance> parseServers(String servers) {
 +    if (servers == null)
 +      return null;
 +    // parse "host:port[INSTANCE]"
 +    Set<TServerInstance> result = new HashSet<TServerInstance>();
 +    if (servers.length() > 0) {
 +      for (String part : servers.split(",")) {
 +        String parts[] = part.split("\\[", 2);
 +        String hostport = parts[0];
 +        String instance = parts[1];
 +        if (instance != null && instance.endsWith("]"))
 +          instance = instance.substring(0, instance.length() - 1);
 +        result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance));
 +      }
 +    }
 +    return result;
 +  }
 +  
 +  private Map<Text,MergeInfo> parseMerges(String merges) {
 +    if (merges == null)
 +      return null;
 +    try {
 +      Map<Text,MergeInfo> result = new HashMap<Text,MergeInfo>();
 +      DataInputBuffer buffer = new DataInputBuffer();
-       byte[] data = Base64.decodeBase64(merges.getBytes(Constants.UTF8));
++      byte[] data = Base64.decodeBase64(merges.getBytes(UTF_8));
 +      buffer.reset(data, data.length);
 +      while (buffer.available() > 0) {
 +        MergeInfo mergeInfo = new MergeInfo();
 +        mergeInfo.readFields(buffer);
 +        result.put(mergeInfo.extent.getTableId(), mergeInfo);
 +      }
 +      return result;
 +    } catch (Exception ex) {
 +      throw new RuntimeException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  protected void consume() throws IOException {
 +    while (getSource().hasTop()) {
 +      Key k = getSource().getTopKey();
 +      Value v = getSource().getTopValue();
 +      
 +      if (onlineTables == null || current == null)
 +        return;
 +      
 +      TabletLocationState tls;
 +      try {
 +        tls = MetaDataTableScanner.createTabletLocationState(k, v);
 +        if (tls == null)
 +          return;
 +      } catch (BadLocationStateException e) {
 +        // maybe the master can do something with a tablet with bad/inconsistent state
 +        return;
 +      }
 +      // we always want data about merges
 +      MergeInfo merge = merges.get(tls.extent.getTableId());
 +      if (merge != null && merge.getExtent() != null && merge.getExtent().overlaps(tls.extent)) {
 +        return;
 +      }
 +      // is the table supposed to be online or offline?
 +      boolean shouldBeOnline = onlineTables.contains(tls.extent.getTableId().toString());
 +      
 +      switch (tls.getState(current)) {
 +        case ASSIGNED:
 +          // we always want data about assigned tablets
 +          return;
 +        case HOSTED:
 +          if (!shouldBeOnline)
 +            return;
 +        case ASSIGNED_TO_DEAD_SERVER:
 +          return;
 +        case UNASSIGNED:
 +          if (shouldBeOnline)
 +            return;
 +      }
 +      // table is in the expected state so don't bother returning any information about it
 +      getSource().next();
 +    }
 +  }
 +  
 +  @Override
 +  public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
 +    throw new UnsupportedOperationException();
 +  }
 +  
 +  public static void setCurrentServers(IteratorSetting cfg, Set<TServerInstance> goodServers) {
 +    if (goodServers != null) {
 +      List<String> servers = new ArrayList<String>();
 +      for (TServerInstance server : goodServers)
 +        servers.add(server.toString());
 +      cfg.addOption(SERVERS_OPTION, StringUtil.join(servers, ","));
 +    }
 +  }
 +  
 +  public static void setOnlineTables(IteratorSetting cfg, Set<String> onlineTables) {
 +    if (onlineTables != null)
 +      cfg.addOption(TABLES_OPTION, StringUtil.join(onlineTables, ","));
 +  }
 +  
 +  public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) {
 +    DataOutputBuffer buffer = new DataOutputBuffer();
 +    try {
 +      for (MergeInfo info : merges) {
 +        KeyExtent extent = info.getExtent();
 +        if (extent != null && !info.getState().equals(MergeState.NONE)) {
 +          info.write(buffer);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      throw new RuntimeException(ex);
 +    }
 +    String encoded = Base64.encodeBase64String(Arrays.copyOf(buffer.getData(), buffer.getLength()));
 +    cfg.addOption(MERGES_OPTION, encoded);
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
index b0ed03f,0000000..8f0c9c7
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
@@@ -1,97 -1,0 +1,98 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +
 +public class ZooStore implements DistributedStore {
 +  
 +  private static final Logger log = Logger.getLogger(ZooStore.class);
 +  
 +  String basePath;
 +  
 +  ZooCache cache = new ZooCache();
 +  
 +  public ZooStore(String basePath) throws IOException {
 +    if (basePath.endsWith("/"))
 +      basePath = basePath.substring(0, basePath.length() - 1);
 +    this.basePath = basePath;
 +  }
 +  
 +  public ZooStore() throws IOException {
 +    this(ZooUtil.getRoot(HdfsZooInstance.getInstance().getInstanceID()));
 +  }
 +  
 +  @Override
 +  public byte[] get(String path) throws DistributedStoreException {
 +    try {
 +      return cache.get(relative(path));
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  private String relative(String path) {
 +    return basePath + path;
 +  }
 +  
 +  @Override
 +  public List<String> getChildren(String path) throws DistributedStoreException {
 +    try {
 +      return cache.getChildren(relative(path));
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  public void put(String path, byte[] bs) throws DistributedStoreException {
 +    try {
 +      path = relative(path);
 +      ZooReaderWriter.getInstance().putPersistentData(path, bs, NodeExistsPolicy.OVERWRITE);
 +      cache.clear();
-       log.debug("Wrote " + new String(bs, Constants.UTF8) + " to " + path);
++      log.debug("Wrote " + new String(bs, UTF_8) + " to " + path);
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  public void remove(String path) throws DistributedStoreException {
 +    try {
 +      log.debug("Removing " + path);
 +      path = relative(path);
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      if (zoo.exists(path))
 +        zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
 +      cache.clear();
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
index 0a0e1ee,0000000..15ade65
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
@@@ -1,179 -1,0 +1,180 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Iterator;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.commons.lang.NotImplementedException;
 +import org.apache.log4j.Logger;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class ZooTabletStateStore extends TabletStateStore {
 +  
 +  private static final Logger log = Logger.getLogger(ZooTabletStateStore.class);
 +  final private DistributedStore store;
 +  
 +  public ZooTabletStateStore(DistributedStore store) {
 +    this.store = store;
 +  }
 +  
 +  public ZooTabletStateStore() throws DistributedStoreException {
 +    try {
 +      store = new ZooStore();
 +    } catch (IOException ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  public ClosableIterator<TabletLocationState> iterator() {
 +    return new ClosableIterator<TabletLocationState>() {
 +      boolean finished = false;
 +      
 +      @Override
 +      public boolean hasNext() {
 +        return !finished;
 +      }
 +      
 +      @Override
 +      public TabletLocationState next() {
 +        finished = true;
 +        try {
 +          byte[] future = store.get(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
 +          byte[] current = store.get(RootTable.ZROOT_TABLET_LOCATION);
 +          byte[] last = store.get(RootTable.ZROOT_TABLET_LAST_LOCATION);
 +          
 +          TServerInstance currentSession = null;
 +          TServerInstance futureSession = null;
 +          TServerInstance lastSession = null;
 +          
 +          if (future != null)
 +            futureSession = parse(future);
 +          
 +          if (last != null)
 +            lastSession = parse(last);
 +          
 +          if (current != null) {
 +            currentSession = parse(current);
 +            futureSession = null;
 +          }
 +          List<Collection<String>> logs = new ArrayList<Collection<String>>();
 +          for (String entry : store.getChildren(RootTable.ZROOT_TABLET_WALOGS)) {
 +            byte[] logInfo = store.get(RootTable.ZROOT_TABLET_WALOGS + "/" + entry);
 +            if (logInfo != null) {
 +              LogEntry logEntry = new LogEntry();
 +              logEntry.fromBytes(logInfo);
 +              logs.add(logEntry.logSet);
 +              log.debug("root tablet logSet " + logEntry.logSet);
 +            }
 +          }
 +          TabletLocationState result = new TabletLocationState(RootTable.EXTENT, futureSession, currentSession, lastSession, logs, false);
 +          log.debug("Returning root tablet state: " + result);
 +          return result;
 +        } catch (Exception ex) {
 +          throw new RuntimeException(ex);
 +        }
 +      }
 +      
 +      @Override
 +      public void remove() {
 +        throw new NotImplementedException();
 +      }
 +
 +      @Override
 +      public void close() {
 +      }
 +    };
 +  }
 +  
 +  protected TServerInstance parse(byte[] current) {
-     String str = new String(current, Constants.UTF8);
++    String str = new String(current, UTF_8);
 +    String[] parts = str.split("[|]", 2);
 +    HostAndPort address = HostAndPort.fromString(parts[0]);
 +    if (parts.length > 1 && parts[1] != null && parts[1].length() > 0) {
 +      return new TServerInstance(address, parts[1]);
 +    } else {
 +      // a 1.2 location specification: DO NOT WANT
 +      return null;
 +    }
 +  }
 +  
 +  @Override
 +  public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException {
 +    if (assignments.size() != 1)
 +      throw new IllegalArgumentException("There is only one root tablet");
 +    Assignment assignment = assignments.iterator().next();
 +    if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
 +      throw new IllegalArgumentException("You can only store the root tablet location");
 +    String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
 +    Iterator<TabletLocationState> currentIter = iterator();
 +    TabletLocationState current = currentIter.next();
 +    if (current.current != null) {
 +      throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
 +    }
-     store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(Constants.UTF8));
++    store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(UTF_8));
 +  }
 +  
 +  @Override
 +  public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException {
 +    if (assignments.size() != 1)
 +      throw new IllegalArgumentException("There is only one root tablet");
 +    Assignment assignment = assignments.iterator().next();
 +    if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
 +      throw new IllegalArgumentException("You can only store the root tablet location");
 +    String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
 +    Iterator<TabletLocationState> currentIter = iterator();
 +    TabletLocationState current = currentIter.next();
 +    if (current.current != null) {
 +      throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
 +    }
 +    if (!current.future.equals(assignment.server)) {
 +      throw new DistributedStoreException("Root tablet is already assigned to " + current.future);
 +    }
-     store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(Constants.UTF8));
-     store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(Constants.UTF8));
++    store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(UTF_8));
++    store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(UTF_8));
 +    // Make the following unnecessary by making the entire update atomic
 +    store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
 +    log.debug("Put down root tablet location");
 +  }
 +  
 +  @Override
 +  public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException {
 +    if (tablets.size() != 1)
 +      throw new IllegalArgumentException("There is only one root tablet");
 +    TabletLocationState tls = tablets.iterator().next();
 +    if (tls.extent.compareTo(RootTable.EXTENT) != 0)
 +      throw new IllegalArgumentException("You can only store the root tablet location");
 +    store.remove(RootTable.ZROOT_TABLET_LOCATION);
 +    store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
 +    log.debug("unassign root tablet location");
 +  }
 +  
 +  @Override
 +  public String name() {
 +    return "Root Table";
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 16b4bb8,0000000..409fa62
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,273 -1,0 +1,274 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.metrics;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.io.IOException;
 +import java.io.OutputStreamWriter;
 +import java.io.Writer;
 +import java.lang.management.ManagementFactory;
 +import java.text.SimpleDateFormat;
 +import java.util.Date;
 +import java.util.concurrent.ConcurrentHashMap;
 +
 +import javax.management.MBeanServer;
 +import javax.management.ObjectName;
 +import javax.management.StandardMBean;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.commons.lang.builder.ToStringBuilder;
 +import org.apache.commons.lang.builder.ToStringStyle;
 +import org.apache.commons.lang.time.DateUtils;
 +
 +public abstract class AbstractMetricsImpl {
 +  
 +  public class Metric {
 +    
 +    private long count = 0;
 +    private long avg = 0;
 +    private long min = 0;
 +    private long max = 0;
 +    
 +    public long getCount() {
 +      return count;
 +    }
 +    
 +    public long getAvg() {
 +      return avg;
 +    }
 +    
 +    public long getMin() {
 +      return min;
 +    }
 +    
 +    public long getMax() {
 +      return max;
 +    }
 +    
 +    public void incCount() {
 +      count++;
 +    }
 +    
 +    public void addAvg(long a) {
 +      if (a < 0)
 +        return;
 +      avg = (long) ((avg * .8) + (a * .2));
 +    }
 +    
 +    public void addMin(long a) {
 +      if (a < 0)
 +        return;
 +      min = Math.min(min, a);
 +    }
 +    
 +    public void addMax(long a) {
 +      if (a < 0)
 +        return;
 +      max = Math.max(max, a);
 +    }
 +    
 +    @Override
 +    public String toString() {
 +      return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
 +    }
 +    
 +  }
 +  
 +  static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
 +  
 +  private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
 +  
 +  private boolean currentlyLogging = false;
 +  
 +  private File logDir = null;
 +  
 +  private String metricsPrefix = null;
 +  
 +  private Date today = new Date();
 +  
 +  private File logFile = null;
 +  
 +  private Writer logWriter = null;
 +  
 +  private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
 +  
 +  private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
 +  
 +  private MetricsConfiguration config = null;
 +  
 +  public AbstractMetricsImpl() {
 +    this.metricsPrefix = getMetricsPrefix();
 +    config = new MetricsConfiguration(metricsPrefix);
 +  }
 +  
 +  /**
 +   * Registers a StandardMBean with the MBean Server
 +   */
 +  public void register(StandardMBean mbean) throws Exception {
 +    // Register this object with the MBeanServer
 +    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 +    if (null == getObjectName())
 +      throw new IllegalArgumentException("MBean object name must be set.");
 +    mbs.registerMBean(mbean, getObjectName());
 +    
 +    setupLogging();
 +  }
 +  
 +  /**
 +   * Registers this MBean with the MBean Server
 +   */
 +  public void register() throws Exception {
 +    // Register this object with the MBeanServer
 +    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 +    if (null == getObjectName())
 +      throw new IllegalArgumentException("MBean object name must be set.");
 +    mbs.registerMBean(this, getObjectName());
 +    setupLogging();
 +  }
 +  
 +  public void createMetric(String name) {
 +    registry.put(name, new Metric());
 +  }
 +  
 +  public Metric getMetric(String name) {
 +    return registry.get(name);
 +  }
 +  
 +  public long getMetricCount(String name) {
 +    return registry.get(name).getCount();
 +  }
 +  
 +  public long getMetricAvg(String name) {
 +    return registry.get(name).getAvg();
 +  }
 +  
 +  public long getMetricMin(String name) {
 +    return registry.get(name).getMin();
 +  }
 +  
 +  public long getMetricMax(String name) {
 +    return registry.get(name).getMax();
 +  }
 +  
 +  private void setupLogging() throws IOException {
 +    if (null == config.getMetricsConfiguration())
 +      return;
 +    // If we are already logging, then return
 +    if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +      // Check to see if directory exists, else make it
 +      String mDir = config.getMetricsConfiguration().getString("logging.dir");
 +      if (null != mDir) {
 +        File dir = new File(mDir);
 +        if (!dir.isDirectory())
 +          if (!dir.mkdir()) 
 +            log.warn("Could not create log directory: " + dir);
 +        logDir = dir;
 +        // Create new log file
 +        startNewLog();
 +      }
 +      currentlyLogging = true;
 +    }
 +  }
 +  
 +  private void startNewLog() throws IOException {
 +    if (null != logWriter) {
 +      logWriter.flush();
 +      logWriter.close();
 +    }
 +    logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
 +    if (!logFile.exists()) {
 +      if (!logFile.createNewFile()) {
 +        log.error("Unable to create new log file");
 +        currentlyLogging = false;
 +        return;
 +      }
 +    }
-     logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
++    logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), UTF_8);
 +  }
 +  
 +  private void writeToLog(String name) throws IOException {
 +    if (null == logWriter)
 +      return;
 +    // Increment the date if we have to
 +    Date now = new Date();
 +    if (!DateUtils.isSameDay(today, now)) {
 +      today = now;
 +      startNewLog();
 +    }
 +    logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
 +  }
 +  
 +  public void add(String name, long time) {
 +    if (isEnabled()) {
 +      registry.get(name).incCount();
 +      registry.get(name).addAvg(time);
 +      registry.get(name).addMin(time);
 +      registry.get(name).addMax(time);
 +      // If we are not currently logging and should be, then initialize
 +      if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +        try {
 +          setupLogging();
 +        } catch (IOException ioe) {
 +          log.error("Error setting up log", ioe);
 +        }
 +      } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +        // if we are currently logging and shouldn't be, then close logs
 +        try {
 +          logWriter.flush();
 +          logWriter.close();
 +          logWriter = null;
 +          logFile = null;
 +        } catch (Exception e) {
 +          log.error("Error stopping metrics logging", e);
 +        }
 +        currentlyLogging = false;
 +      }
 +      if (currentlyLogging) {
 +        try {
 +          writeToLog(name);
 +        } catch (IOException ioe) {
 +          log.error("Error writing to metrics log", ioe);
 +        }
 +      }
 +    }
 +  }
 +  
 +  public boolean isEnabled() {
 +    return config.isEnabled();
 +  }
 +  
 +  protected abstract ObjectName getObjectName();
 +  
 +  protected abstract String getMetricsPrefix();
 +  
 +  @Override
 +  protected void finalize() {
 +    if (null != logWriter) {
 +      try {
 +        logWriter.close();
 +      } catch (Exception e) {
 +        // do nothing
 +      } finally {
 +        logWriter = null;
 +      }
 +    }
 +    logFile = null;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
index 0a5341a,0000000..4e909c0
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
@@@ -1,187 -1,0 +1,189 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.monitor;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.net.ServerSocket;
 +import java.net.Socket;
 +import java.util.ArrayList;
 +import java.util.LinkedHashMap;
 +import java.util.List;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.LogManager;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.net.SocketNode;
 +import org.apache.log4j.spi.LoggingEvent;
 +import org.apache.zookeeper.KeeperException;
 +
 +/**
 + * Hijack log4j and capture log events for display.
 + * 
 + */
 +public class LogService extends org.apache.log4j.AppenderSkeleton {
 +
 +  private static final Logger log = Logger.getLogger(LogService.class);
 +
 +  /**
 +   * Read logging events forward to us over the net.
 +   * 
 +   */
 +  static class SocketServer implements Runnable {
 +    private ServerSocket server = null;
 +
 +    public SocketServer(int port) {
 +      try {
 +        server = new ServerSocket(port);
 +      } catch (IOException io) {
 +        throw new RuntimeException(io);
 +      }
 +    }
 +
 +    public int getLocalPort() {
 +      return server.getLocalPort();
 +    }
 +
 +    @Override
 +    public void run() {
 +      try {
 +        while (true) {
 +          log.debug("Waiting for log message senders");
 +          Socket socket = server.accept();
 +          log.debug("Got a new connection");
 +          Thread t = new Daemon(new SocketNode(socket, LogManager.getLoggerRepository()));
 +          t.start();
 +        }
 +      } catch (IOException io) {
 +        log.error(io, io);
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
 +   * 
 +   * @param conf
 +   *          configuration for the instance
 +   * @param instanceId
 +   *          instanceId for the instance
 +   * @param hostAddress
 +   *          Address that monitor process is bound to
 +   */
 +  public static void startLogListener(AccumuloConfiguration conf, String instanceId, String hostAddress) {
 +    try {
 +      SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));
 +
 +      // getLocalPort will return the actual ephemeral port used when '0' was provided.
 +      String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
 +
 +      log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);
 +
 +      final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
 +      final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +      // Delete before we try to re-create in case the previous session hasn't yet expired
 +      try {
 +        zoo.delete(path, -1);
 +      } catch (KeeperException e) {
 +        // We don't care if the node is already gone
 +        if (!KeeperException.Code.NONODE.equals(e.code())) {
 +          throw e;
 +        }
 +      }
 +
-       zoo.putEphemeralData(path, logForwardingAddr.getBytes(Constants.UTF8));
++      zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
 +
 +      new Daemon(server).start();
 +    } catch (Throwable t) {
 +      log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
 +    }
 +  }
 +
 +  static private LogService instance = null;
 +
 +  synchronized public static LogService getInstance() {
 +    if (instance == null)
 +      return new LogService();
 +    return instance;
 +  }
 +
 +  private static final int MAX_LOGS = 50;
 +
 +  private LinkedHashMap<String,DedupedLogEvent> events = new LinkedHashMap<String,DedupedLogEvent>(MAX_LOGS + 1, (float) .75, true) {
 +
 +    private static final long serialVersionUID = 1L;
 +
 +    @Override
 +    @SuppressWarnings("rawtypes")
 +    protected boolean removeEldestEntry(Map.Entry eldest) {
 +      return size() > MAX_LOGS;
 +    }
 +  };
 +
 +  public LogService() {
 +    synchronized (LogService.class) {
 +      instance = this;
 +    }
 +  }
 +
 +  @Override
 +  synchronized protected void append(LoggingEvent ev) {
 +    Object application = ev.getMDC("application");
 +    if (application == null || application.toString().isEmpty())
 +      return;
 +
 +    DedupedLogEvent dev = new DedupedLogEvent(ev);
 +
 +    // if event is present, increase the count
 +    if (events.containsKey(dev.toString())) {
 +      DedupedLogEvent oldDev = events.remove(dev.toString());
 +      dev.setCount(oldDev.getCount() + 1);
 +    }
 +    events.put(dev.toString(), dev);
 +  }
 +
 +  @Override
 +  public void close() {
 +    events = null;
 +  }
 +
 +  @Override
 +  public synchronized void doAppend(LoggingEvent event) {
 +    super.doAppend(event);
 +  }
 +
 +  @Override
 +  public boolean requiresLayout() {
 +    return false;
 +  }
 +
 +  synchronized public List<DedupedLogEvent> getEvents() {
 +    return new ArrayList<DedupedLogEvent>(events.values());
 +  }
 +
 +  synchronized public void clear() {
 +    events.clear();
 +  }
 +}


[26/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 03fe069,0000000..94be0bb
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -1,3928 -1,0 +1,3929 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.lang.management.GarbageCollectorMXBean;
 +import java.lang.management.ManagementFactory;
 +import java.net.Socket;
 +import java.net.UnknownHostException;
 +import java.nio.ByteBuffer;
 +import java.security.SecureRandom;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.EnumMap;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.SortedSet;
 +import java.util.TimerTask;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.concurrent.ArrayBlockingQueue;
 +import java.util.concurrent.BlockingDeque;
 +import java.util.concurrent.CancellationException;
 +import java.util.concurrent.ExecutionException;
 +import java.util.concurrent.LinkedBlockingDeque;
 +import java.util.concurrent.RunnableFuture;
 +import java.util.concurrent.ThreadPoolExecutor;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.TimeoutException;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicInteger;
 +import java.util.concurrent.atomic.AtomicLong;
 +import java.util.concurrent.atomic.AtomicReference;
 +
 +import javax.management.ObjectName;
 +import javax.management.StandardMBean;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.CompressedIterators;
 +import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.TabletType;
 +import org.apache.accumulo.core.client.impl.Translator;
 +import org.apache.accumulo.core.client.impl.Translator.TKeyExtentTranslator;
 +import org.apache.accumulo.core.client.impl.Translator.TRangeTranslator;
 +import org.apache.accumulo.core.client.impl.Translators;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Constraint.Environment;
 +import org.apache.accumulo.core.constraints.Violations;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Column;
 +import org.apache.accumulo.core.data.ConstraintViolationSummary;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.thrift.InitialMultiScan;
 +import org.apache.accumulo.core.data.thrift.InitialScan;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.MapFileInfo;
 +import org.apache.accumulo.core.data.thrift.MultiScanResult;
 +import org.apache.accumulo.core.data.thrift.ScanResult;
 +import org.apache.accumulo.core.data.thrift.TCMResult;
 +import org.apache.accumulo.core.data.thrift.TCMStatus;
 +import org.apache.accumulo.core.data.thrift.TColumn;
 +import org.apache.accumulo.core.data.thrift.TCondition;
 +import org.apache.accumulo.core.data.thrift.TConditionalMutation;
 +import org.apache.accumulo.core.data.thrift.TConditionalSession;
 +import org.apache.accumulo.core.data.thrift.TKey;
 +import org.apache.accumulo.core.data.thrift.TKeyExtent;
 +import org.apache.accumulo.core.data.thrift.TKeyValue;
 +import org.apache.accumulo.core.data.thrift.TMutation;
 +import org.apache.accumulo.core.data.thrift.TRange;
 +import org.apache.accumulo.core.data.thrift.UpdateErrors;
 +import org.apache.accumulo.core.iterators.IterationInterruptedException;
 +import org.apache.accumulo.core.master.thrift.Compacting;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletLoadState;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.AuthorizationContainer;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
 +import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
 +import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
 +import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
 +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 +import org.apache.accumulo.core.tabletserver.thrift.ScanState;
 +import org.apache.accumulo.core.tabletserver.thrift.ScanType;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 +import org.apache.accumulo.core.util.ByteBufferUtil;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ServerServices.Service;
 +import org.apache.accumulo.core.util.SimpleThreadPool;
 +import org.apache.accumulo.core.util.Stat;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.ClientServiceHandler;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.data.ServerMutation;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.log.SortedLogState;
 +import org.apache.accumulo.server.master.recovery.RecoveryPath;
 +import org.apache.accumulo.server.master.state.Assignment;
 +import org.apache.accumulo.server.master.state.DistributedStoreException;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.state.TabletLocationState;
 +import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
 +import org.apache.accumulo.server.master.state.TabletStateStore;
 +import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 +import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
 +import org.apache.accumulo.server.problems.ProblemReport;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.FileSystemMonitor;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.RpcWrapper;
 +import org.apache.accumulo.server.util.TServerUtils;
 +import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
 +import org.apache.accumulo.server.util.time.RelativeTime;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.start.classloader.vfs.ContextManager;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.instrument.Trace;
 +import org.apache.accumulo.trace.thrift.TInfo;
 +import org.apache.accumulo.tserver.Compactor.CompactionInfo;
 +import org.apache.accumulo.tserver.RowLocks.RowLock;
 +import org.apache.accumulo.tserver.Tablet.CommitSession;
 +import org.apache.accumulo.tserver.Tablet.KVEntry;
 +import org.apache.accumulo.tserver.Tablet.LookupResult;
 +import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
 +import org.apache.accumulo.tserver.Tablet.ScanBatch;
 +import org.apache.accumulo.tserver.Tablet.Scanner;
 +import org.apache.accumulo.tserver.Tablet.SplitInfo;
 +import org.apache.accumulo.tserver.Tablet.TConstraintViolationException;
 +import org.apache.accumulo.tserver.Tablet.TabletClosedException;
 +import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
 +import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
 +import org.apache.accumulo.tserver.data.ServerConditionalMutation;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.LogSorter;
 +import org.apache.accumulo.tserver.log.MutationReceiver;
 +import org.apache.accumulo.tserver.log.TabletServerLogger;
 +import org.apache.accumulo.tserver.mastermessage.MasterMessage;
 +import org.apache.accumulo.tserver.mastermessage.SplitReportMessage;
 +import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
 +import org.apache.accumulo.tserver.metrics.TabletServerMBean;
 +import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
 +import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
 +import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
 +import org.apache.commons.collections.map.LRUMap;
 +import org.apache.hadoop.fs.FSError;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.TProcessor;
 +import org.apache.thrift.TServiceClient;
 +import org.apache.thrift.server.TServer;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +enum ScanRunState {
 +  QUEUED, RUNNING, FINISHED
 +}
 +
 +public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.tserver.metrics.TabletServerMBean {
 +  private static final Logger log = Logger.getLogger(TabletServer.class);
 +
 +  private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
 +  private static long lastMemorySize = 0;
 +  private static long lastMemoryCheckTime = 0;
 +  private static long gcTimeIncreasedCount;
 +
 +  private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
 +  private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000;
 +  private static final long TIME_BETWEEN_GC_CHECKS = 5000;
 +
 +  private TabletServerLogger logger;
 +
 +  protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
 +
 +  private ServerConfiguration serverConfig;
 +  private LogSorter logSorter = null;
 +
 +  public TabletServer(ServerConfiguration conf, VolumeManager fs) {
 +    super();
 +    this.serverConfig = conf;
 +    this.instance = conf.getInstance();
 +    this.fs = fs;
 +    this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        synchronized (onlineTablets) {
 +          long now = System.currentTimeMillis();
 +          for (Tablet tablet : onlineTablets.values())
 +            try {
 +              tablet.updateRates(now);
 +            } catch (Exception ex) {
 +              log.error(ex, ex);
 +            }
 +        }
 +      }
 +    }, TIME_BETWEEN_GC_CHECKS, TIME_BETWEEN_GC_CHECKS);
 +  }
 +
 +  private synchronized static void logGCInfo(AccumuloConfiguration conf) {
 +    long now = System.currentTimeMillis();
 +
 +    List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
 +    Runtime rt = Runtime.getRuntime();
 +
 +    StringBuilder sb = new StringBuilder("gc");
 +
 +    boolean sawChange = false;
 +
 +    long maxIncreaseInCollectionTime = 0;
 +
 +    for (GarbageCollectorMXBean gcBean : gcmBeans) {
 +      Long prevTime = prevGcTime.get(gcBean.getName());
 +      long pt = 0;
 +      if (prevTime != null) {
 +        pt = prevTime;
 +      }
 +
 +      long time = gcBean.getCollectionTime();
 +
 +      if (time - pt != 0) {
 +        sawChange = true;
 +      }
 +
 +      long increaseInCollectionTime = time - pt;
 +      sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
 +      maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
 +      prevGcTime.put(gcBean.getName(), time);
 +    }
 +
 +    long mem = rt.freeMemory();
 +    if (maxIncreaseInCollectionTime == 0) {
 +      gcTimeIncreasedCount = 0;
 +    } else {
 +      gcTimeIncreasedCount++;
 +      if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
 +        log.warn("Running low on memory");
 +        gcTimeIncreasedCount = 0;
 +      }
 +    }
 +
 +    if (mem > lastMemorySize) {
 +      sawChange = true;
 +    }
 +
 +    String sign = "+";
 +    if (mem - lastMemorySize <= 0) {
 +      sign = "";
 +    }
 +
 +    sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
 +
 +    if (sawChange) {
 +      log.debug(sb.toString());
 +    }
 +
 +    final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
 +    if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) {
 +      long diff = now - lastMemoryCheckTime;
 +      if (diff > keepAliveTimeout) {
 +        log.warn(String.format("GC pause checker not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check",
 +                    TIME_BETWEEN_GC_CHECKS / 1000., diff / 1000.));
 +      }
 +      lastMemoryCheckTime = now;
 +      return;
 +    }
 +
 +    if (maxIncreaseInCollectionTime > keepAliveTimeout) {
 +      Halt.halt("Garbage collection may be interfering with lock keep-alive.  Halting.", -1);
 +    }
 +
 +    lastMemorySize = mem;
 +    lastMemoryCheckTime = now;
 +  }
 +
 +  private TabletStatsKeeper statsKeeper;
 +
 +  private static class Session {
 +    long lastAccessTime;
 +    long startTime;
 +    String user;
 +    String client = TServerUtils.clientAddress.get();
 +    public boolean reserved;
 +
 +    public void cleanup() {}
 +  }
 +
 +  private static class SessionManager {
 +
 +    SecureRandom random;
 +    Map<Long,Session> sessions;
 +    long maxIdle;
 +
 +    SessionManager(AccumuloConfiguration conf) {
 +      random = new SecureRandom();
 +      sessions = new HashMap<Long,Session>();
 +
 +      maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
 +
 +      Runnable r = new Runnable() {
 +        @Override
 +        public void run() {
 +          sweep(maxIdle);
 +        }
 +      };
 +
 +      SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
 +    }
 +
 +    synchronized long createSession(Session session, boolean reserve) {
 +      long sid = random.nextLong();
 +
 +      while (sessions.containsKey(sid)) {
 +        sid = random.nextLong();
 +      }
 +
 +      sessions.put(sid, session);
 +
 +      session.reserved = reserve;
 +
 +      session.startTime = session.lastAccessTime = System.currentTimeMillis();
 +
 +      return sid;
 +    }
 +
 +    long getMaxIdleTime() {
 +      return maxIdle;
 +    }
 +
 +    /**
 +     * while a session is reserved, it cannot be canceled or removed
 +     */
 +    synchronized Session reserveSession(long sessionId) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null) {
 +        if (session.reserved)
 +          throw new IllegalStateException();
 +        session.reserved = true;
 +      }
 +
 +      return session;
 +
 +    }
 +
 +    synchronized Session reserveSession(long sessionId, boolean wait) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null) {
 +        while (wait && session.reserved) {
 +          try {
 +            wait(1000);
 +          } catch (InterruptedException e) {
 +            throw new RuntimeException();
 +          }
 +        }
 +
 +        if (session.reserved)
 +          throw new IllegalStateException();
 +        session.reserved = true;
 +      }
 +
 +      return session;
 +
 +    }
 +
 +    synchronized void unreserveSession(Session session) {
 +      if (!session.reserved)
 +        throw new IllegalStateException();
 +      notifyAll();
 +      session.reserved = false;
 +      session.lastAccessTime = System.currentTimeMillis();
 +    }
 +
 +    synchronized void unreserveSession(long sessionId) {
 +      Session session = getSession(sessionId);
 +      if (session != null)
 +        unreserveSession(session);
 +    }
 +
 +    synchronized Session getSession(long sessionId) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null)
 +        session.lastAccessTime = System.currentTimeMillis();
 +      return session;
 +    }
 +
 +    Session removeSession(long sessionId) {
 +      return removeSession(sessionId, false);
 +    }
 +
 +    Session removeSession(long sessionId, boolean unreserve) {
 +      Session session = null;
 +      synchronized (this) {
 +        session = sessions.remove(sessionId);
 +        if (unreserve && session != null)
 +          unreserveSession(session);
 +      }
 +
 +      // do clean up out side of lock..
 +      if (session != null)
 +        session.cleanup();
 +
 +      return session;
 +    }
 +
 +    private void sweep(long maxIdle) {
 +      ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
 +      synchronized (this) {
 +        Iterator<Session> iter = sessions.values().iterator();
 +        while (iter.hasNext()) {
 +          Session session = iter.next();
 +          long idleTime = System.currentTimeMillis() - session.lastAccessTime;
 +          if (idleTime > maxIdle && !session.reserved) {
 +            iter.remove();
 +            sessionsToCleanup.add(session);
 +          }
 +        }
 +      }
 +
 +      // do clean up outside of lock
 +      for (Session session : sessionsToCleanup) {
 +        session.cleanup();
 +      }
 +    }
 +
 +    synchronized void removeIfNotAccessed(final long sessionId, long delay) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null) {
 +        final long removeTime = session.lastAccessTime;
 +        TimerTask r = new TimerTask() {
 +          @Override
 +          public void run() {
 +            Session sessionToCleanup = null;
 +            synchronized (SessionManager.this) {
 +              Session session2 = sessions.get(sessionId);
 +              if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
 +                sessions.remove(sessionId);
 +                sessionToCleanup = session2;
 +              }
 +            }
 +
 +            // call clean up outside of lock
 +            if (sessionToCleanup != null)
 +              sessionToCleanup.cleanup();
 +          }
 +        };
 +
 +        SimpleTimer.getInstance().schedule(r, delay);
 +      }
 +    }
 +
 +    public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
 +      Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
 +      for (Entry<Long,Session> entry : sessions.entrySet()) {
 +
 +        Session session = entry.getValue();
 +        @SuppressWarnings("rawtypes")
 +        ScanTask nbt = null;
 +        String tableID = null;
 +
 +        if (session instanceof ScanSession) {
 +          ScanSession ss = (ScanSession) session;
 +          nbt = ss.nextBatchTask;
 +          tableID = ss.extent.getTableId().toString();
 +        } else if (session instanceof MultiScanSession) {
 +          MultiScanSession mss = (MultiScanSession) session;
 +          nbt = mss.lookupTask;
 +          tableID = mss.threadPoolExtent.getTableId().toString();
 +        }
 +
 +        if (nbt == null)
 +          continue;
 +
 +        ScanRunState srs = nbt.getScanRunState();
 +
 +        if (srs == ScanRunState.FINISHED)
 +          continue;
 +
 +        MapCounter<ScanRunState> stateCounts = counts.get(tableID);
 +        if (stateCounts == null) {
 +          stateCounts = new MapCounter<ScanRunState>();
 +          counts.put(tableID, stateCounts);
 +        }
 +
 +        stateCounts.increment(srs, 1);
 +      }
 +
 +      return counts;
 +    }
 +
 +    public synchronized List<ActiveScan> getActiveScans() {
 +
 +      ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
 +
 +      long ct = System.currentTimeMillis();
 +
 +      for (Entry<Long,Session> entry : sessions.entrySet()) {
 +        Session session = entry.getValue();
 +        if (session instanceof ScanSession) {
 +          ScanSession ss = (ScanSession) session;
 +
 +          ScanState state = ScanState.RUNNING;
 +
 +          ScanTask<ScanBatch> nbt = ss.nextBatchTask;
 +          if (nbt == null) {
 +            state = ScanState.IDLE;
 +          } else {
 +            switch (nbt.getScanRunState()) {
 +              case QUEUED:
 +                state = ScanState.QUEUED;
 +                break;
 +              case FINISHED:
 +                state = ScanState.IDLE;
 +                break;
 +              case RUNNING:
 +              default:
 +                /* do nothing */
 +                break;
 +            }
 +          }
 +
 +          activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
 +              state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
 +
 +        } else if (session instanceof MultiScanSession) {
 +          MultiScanSession mss = (MultiScanSession) session;
 +
 +          ScanState state = ScanState.RUNNING;
 +
 +          ScanTask<MultiScanResult> nbt = mss.lookupTask;
 +          if (nbt == null) {
 +            state = ScanState.IDLE;
 +          } else {
 +            switch (nbt.getScanRunState()) {
 +              case QUEUED:
 +                state = ScanState.QUEUED;
 +                break;
 +              case FINISHED:
 +                state = ScanState.IDLE;
 +                break;
 +              case RUNNING:
 +              default:
 +                /* do nothing */
 +                break;
 +            }
 +          }
 +
 +          activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
 +              ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
 +                  .getAuthorizationsBB()));
 +        }
 +      }
 +
 +      return activeScans;
 +    }
 +  }
 +
 +  static class TservConstraintEnv implements Environment {
 +
 +    private TCredentials credentials;
 +    private SecurityOperation security;
 +    private Authorizations auths;
 +    private KeyExtent ke;
 +
 +    TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
 +      this.security = secOp;
 +      this.credentials = credentials;
 +    }
 +
 +    void setExtent(KeyExtent ke) {
 +      this.ke = ke;
 +    }
 +
 +    @Override
 +    public KeyExtent getExtent() {
 +      return ke;
 +    }
 +
 +    @Override
 +    public String getUser() {
 +      return credentials.getPrincipal();
 +    }
 +
 +    @Override
 +    @Deprecated
 +    public Authorizations getAuthorizations() {
 +      if (auths == null)
 +        try {
 +          this.auths = security.getUserAuthorizations(credentials);
 +        } catch (ThriftSecurityException e) {
 +          throw new RuntimeException(e);
 +        }
 +      return auths;
 +    }
 +
 +    @Override
 +    public AuthorizationContainer getAuthorizationsContainer() {
 +      return new AuthorizationContainer() {
 +        @Override
 +        public boolean contains(ByteSequence auth) {
 +          try {
 +            return security.userHasAuthorizations(credentials,
 +                Collections.<ByteBuffer> singletonList(ByteBuffer.wrap(auth.getBackingArray(), auth.offset(), auth.length())));
 +          } catch (ThriftSecurityException e) {
 +            throw new RuntimeException(e);
 +          }
 +        }
 +      };
 +    }
 +  }
 +
 +  private abstract class ScanTask<T> implements RunnableFuture<T> {
 +
 +    protected AtomicBoolean interruptFlag;
 +    protected ArrayBlockingQueue<Object> resultQueue;
 +    protected AtomicInteger state;
 +    protected AtomicReference<ScanRunState> runState;
 +
 +    private static final int INITIAL = 1;
 +    private static final int ADDED = 2;
 +    private static final int CANCELED = 3;
 +
 +    ScanTask() {
 +      interruptFlag = new AtomicBoolean(false);
 +      runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
 +      state = new AtomicInteger(INITIAL);
 +      resultQueue = new ArrayBlockingQueue<Object>(1);
 +    }
 +
 +    protected void addResult(Object o) {
 +      if (state.compareAndSet(INITIAL, ADDED))
 +        resultQueue.add(o);
 +      else if (state.get() == ADDED)
 +        throw new IllegalStateException("Tried to add more than one result");
 +    }
 +
 +    @Override
 +    public boolean cancel(boolean mayInterruptIfRunning) {
 +      if (!mayInterruptIfRunning)
 +        throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
 +
 +      if (state.get() == CANCELED)
 +        return true;
 +
 +      if (state.compareAndSet(INITIAL, CANCELED)) {
 +        interruptFlag.set(true);
 +        resultQueue = null;
 +        return true;
 +      }
 +
 +      return false;
 +    }
 +
 +    @Override
 +    public T get() throws InterruptedException, ExecutionException {
 +      throw new UnsupportedOperationException();
 +    }
 +
 +    @SuppressWarnings("unchecked")
 +    @Override
 +    public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
 +
 +      ArrayBlockingQueue<Object> localRQ = resultQueue;
 +
 +      if (state.get() == CANCELED)
 +        throw new CancellationException();
 +
 +      if (localRQ == null && state.get() == ADDED)
 +        throw new IllegalStateException("Tried to get result twice");
 +
 +      Object r = localRQ.poll(timeout, unit);
 +
 +      // could have been canceled while waiting
 +      if (state.get() == CANCELED) {
 +        if (r != null)
 +          throw new IllegalStateException("Nothing should have been added when in canceled state");
 +
 +        throw new CancellationException();
 +      }
 +
 +      if (r == null)
 +        throw new TimeoutException();
 +
 +      // make this method stop working now that something is being
 +      // returned
 +      resultQueue = null;
 +
 +      if (r instanceof Throwable)
 +        throw new ExecutionException((Throwable) r);
 +
 +      return (T) r;
 +    }
 +
 +    @Override
 +    public boolean isCancelled() {
 +      return state.get() == CANCELED;
 +    }
 +
 +    @Override
 +    public boolean isDone() {
 +      return runState.get().equals(ScanRunState.FINISHED);
 +    }
 +
 +    public ScanRunState getScanRunState() {
 +      return runState.get();
 +    }
 +
 +  }
 +
 +  private static class ConditionalSession extends Session {
 +    public TCredentials credentials;
 +    public Authorizations auths;
 +    public String tableId;
 +    public AtomicBoolean interruptFlag;
 +
 +    @Override
 +    public void cleanup() {
 +      interruptFlag.set(true);
 +    }
 +  }
 +
 +  private static class UpdateSession extends Session {
 +    public Tablet currentTablet;
 +    public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
 +    Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
 +    HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
 +    public Violations violations;
 +    public TCredentials credentials;
 +    public long totalUpdates = 0;
 +    public long flushTime = 0;
 +    Stat prepareTimes = new Stat();
 +    Stat walogTimes = new Stat();
 +    Stat commitTimes = new Stat();
 +    Stat authTimes = new Stat();
 +    public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
 +    public long queuedMutationSize = 0;
 +    TservConstraintEnv cenv = null;
 +  }
 +
 +  private static class ScanSession extends Session {
 +    public KeyExtent extent;
 +    public HashSet<Column> columnSet;
 +    public List<IterInfo> ssiList;
 +    public Map<String,Map<String,String>> ssio;
 +    public Authorizations auths;
 +    public long entriesReturned = 0;
 +    public Stat nbTimes = new Stat();
 +    public long batchCount = 0;
 +    public volatile ScanTask<ScanBatch> nextBatchTask;
 +    public AtomicBoolean interruptFlag;
 +    public Scanner scanner;
 +    public long readaheadThreshold = Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
 +
 +    @Override
 +    public void cleanup() {
 +      try {
 +        if (nextBatchTask != null)
 +          nextBatchTask.cancel(true);
 +      } finally {
 +        if (scanner != null)
 +          scanner.close();
 +      }
 +    }
 +
 +  }
 +
 +  private static class MultiScanSession extends Session {
 +    HashSet<Column> columnSet;
 +    Map<KeyExtent,List<Range>> queries;
 +    public List<IterInfo> ssiList;
 +    public Map<String,Map<String,String>> ssio;
 +    public Authorizations auths;
 +
 +    // stats
 +    int numRanges;
 +    int numTablets;
 +    int numEntries;
 +    long totalLookupTime;
 +
 +    public volatile ScanTask<MultiScanResult> lookupTask;
 +    public KeyExtent threadPoolExtent;
 +
 +    @Override
 +    public void cleanup() {
 +      if (lookupTask != null)
 +        lookupTask.cancel(true);
 +    }
 +  }
 +
 +  /**
 +   * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
 +   * are monotonically increasing.
 +   *
 +   */
 +  static class WriteTracker {
 +    private static AtomicLong operationCounter = new AtomicLong(1);
 +    private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
 +
 +    WriteTracker() {
 +      for (TabletType ttype : TabletType.values()) {
 +        inProgressWrites.put(ttype, new TreeSet<Long>());
 +      }
 +    }
 +
 +    synchronized long startWrite(TabletType ttype) {
 +      long operationId = operationCounter.getAndIncrement();
 +      inProgressWrites.get(ttype).add(operationId);
 +      return operationId;
 +    }
 +
 +    synchronized void finishWrite(long operationId) {
 +      if (operationId == -1)
 +        return;
 +
 +      boolean removed = false;
 +
 +      for (TabletType ttype : TabletType.values()) {
 +        removed = inProgressWrites.get(ttype).remove(operationId);
 +        if (removed)
 +          break;
 +      }
 +
 +      if (!removed) {
 +        throw new IllegalArgumentException("Attempted to finish write not in progress,  operationId " + operationId);
 +      }
 +
 +      this.notifyAll();
 +    }
 +
 +    synchronized void waitForWrites(TabletType ttype) {
 +      long operationId = operationCounter.getAndIncrement();
 +      while (inProgressWrites.get(ttype).floor(operationId) != null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {
 +          log.error(e, e);
 +        }
 +      }
 +    }
 +
 +    public long startWrite(Set<Tablet> keySet) {
 +      if (keySet.size() == 0)
 +        return -1;
 +
 +      ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
 +
 +      for (Tablet tablet : keySet)
 +        extents.add(tablet.getExtent());
 +
 +      return startWrite(TabletType.type(extents));
 +    }
 +  }
 +
 +  public AccumuloConfiguration getSystemConfiguration() {
 +    return serverConfig.getConfiguration();
 +  }
 +
 +  TransactionWatcher watcher = new TransactionWatcher();
 +
 +  private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
 +
 +    SessionManager sessionManager;
 +
 +    AccumuloConfiguration acuConf = getSystemConfiguration();
 +
 +    TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
 +
 +    TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
 +
 +    WriteTracker writeTracker = new WriteTracker();
 +
 +    private RowLocks rowLocks = new RowLocks();
 +
 +    ThriftClientHandler() {
 +      super(instance, watcher, fs);
 +      log.debug(ThriftClientHandler.class.getName() + " created");
 +      sessionManager = new SessionManager(getSystemConfiguration());
 +      // Register the metrics MBean
 +      try {
 +        updateMetrics.register();
 +        scanMetrics.register();
 +      } catch (Exception e) {
 +        log.error("Exception registering MBean with MBean Server", e);
 +      }
 +    }
 +
 +    @Override
 +    public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
 +        throws ThriftSecurityException {
 +
 +      if (!security.canPerformSystemActions(credentials))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
 +
 +      for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
 +        TKeyExtent tke = entry.getKey();
 +        Map<String,MapFileInfo> fileMap = entry.getValue();
 +        Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
 +        for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
 +          Path path = new Path(mapping.getKey());
 +          FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
 +          path = ns.makeQualified(path);
 +          fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
 +        }
 +
 +        Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
 +
 +        if (importTablet == null) {
 +          failures.add(tke);
 +        } else {
 +          try {
 +            importTablet.importMapFiles(tid, fileRefMap, setTime);
 +          } catch (IOException ioe) {
 +            log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
 +            failures.add(tke);
 +          }
 +        }
 +      }
 +      return failures;
 +    }
 +
 +    private class NextBatchTask extends ScanTask<ScanBatch> {
 +
 +      private long scanID;
 +
 +      NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
 +        this.scanID = scanID;
 +        this.interruptFlag = interruptFlag;
 +
 +        if (interruptFlag.get())
 +          cancel(true);
 +      }
 +
 +      @Override
 +      public void run() {
 +
 +        final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
 +        String oldThreadName = Thread.currentThread().getName();
 +
 +        try {
 +          if (isCancelled() || scanSession == null)
 +            return;
 +
 +          runState.set(ScanRunState.RUNNING);
 +
 +          Thread.currentThread().setName(
 +              "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
 +
 +          Tablet tablet = onlineTablets.get(scanSession.extent);
 +
 +          if (tablet == null) {
 +            addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
 +            return;
 +          }
 +
 +          long t1 = System.currentTimeMillis();
 +          ScanBatch batch = scanSession.scanner.read();
 +          long t2 = System.currentTimeMillis();
 +          scanSession.nbTimes.addStat(t2 - t1);
 +
 +          // there should only be one thing on the queue at a time, so
 +          // it should be ok to call add()
 +          // instead of put()... if add() fails because queue is at
 +          // capacity it means there is code
 +          // problem somewhere
 +          addResult(batch);
 +        } catch (TabletClosedException e) {
 +          addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
 +        } catch (IterationInterruptedException iie) {
 +          if (!isCancelled()) {
 +            log.warn("Iteration interrupted, when scan not cancelled", iie);
 +            addResult(iie);
 +          }
 +        } catch (TooManyFilesException tmfe) {
 +          addResult(tmfe);
 +        } catch (Throwable e) {
 +          log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
 +          addResult(e);
 +        } finally {
 +          runState.set(ScanRunState.FINISHED);
 +          Thread.currentThread().setName(oldThreadName);
 +        }
 +
 +      }
 +    }
 +
 +    private class LookupTask extends ScanTask<MultiScanResult> {
 +
 +      private long scanID;
 +
 +      LookupTask(long scanID) {
 +        this.scanID = scanID;
 +      }
 +
 +      @Override
 +      public void run() {
 +        MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
 +        String oldThreadName = Thread.currentThread().getName();
 +
 +        try {
 +          if (isCancelled() || session == null)
 +            return;
 +
 +          TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
 +          long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
 +
 +          runState.set(ScanRunState.RUNNING);
 +          Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
 +
 +          long bytesAdded = 0;
 +          long maxScanTime = 4000;
 +
 +          long startTime = System.currentTimeMillis();
 +
 +          ArrayList<KVEntry> results = new ArrayList<KVEntry>();
 +          Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
 +          ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
 +          KeyExtent partScan = null;
 +          Key partNextKey = null;
 +          boolean partNextKeyInclusive = false;
 +
 +          Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
 +
 +          // check the time so that the read ahead thread is not monopolized
 +          while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
 +            Entry<KeyExtent,List<Range>> entry = iter.next();
 +
 +            iter.remove();
 +
 +            // check that tablet server is serving requested tablet
 +            Tablet tablet = onlineTablets.get(entry.getKey());
 +            if (tablet == null) {
 +              failures.put(entry.getKey(), entry.getValue());
 +              continue;
 +            }
 +            Thread.currentThread().setName(
 +                "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
 +
 +            LookupResult lookupResult;
 +            try {
 +
 +              // do the following check to avoid a race condition
 +              // between setting false below and the task being
 +              // canceled
 +              if (isCancelled())
 +                interruptFlag.set(true);
 +
 +              lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
 +                  session.ssio, interruptFlag);
 +
 +              // if the tablet was closed it it possible that the
 +              // interrupt flag was set.... do not want it set for
 +              // the next
 +              // lookup
 +              interruptFlag.set(false);
 +
 +            } catch (IOException e) {
 +              log.warn("lookup failed for tablet " + entry.getKey(), e);
 +              throw new RuntimeException(e);
 +            }
 +
 +            bytesAdded += lookupResult.bytesAdded;
 +
 +            if (lookupResult.unfinishedRanges.size() > 0) {
 +              if (lookupResult.closed) {
 +                failures.put(entry.getKey(), lookupResult.unfinishedRanges);
 +              } else {
 +                session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
 +                partScan = entry.getKey();
 +                partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
 +                partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
 +              }
 +            } else {
 +              fullScans.add(entry.getKey());
 +            }
 +          }
 +
 +          long finishTime = System.currentTimeMillis();
 +          session.totalLookupTime += (finishTime - startTime);
 +          session.numEntries += results.size();
 +
 +          // convert everything to thrift before adding result
 +          List<TKeyValue> retResults = new ArrayList<TKeyValue>();
 +          for (KVEntry entry : results)
 +            retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
 +          Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
 +          List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
 +          TKeyExtent retPartScan = null;
 +          TKey retPartNextKey = null;
 +          if (partScan != null) {
 +            retPartScan = partScan.toThrift();
 +            retPartNextKey = partNextKey.toThrift();
 +          }
 +          // add results to queue
 +          addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
 +        } catch (IterationInterruptedException iie) {
 +          if (!isCancelled()) {
 +            log.warn("Iteration interrupted, when scan not cancelled", iie);
 +            addResult(iie);
 +          }
 +        } catch (Throwable e) {
 +          log.warn("exception while doing multi-scan ", e);
 +          addResult(e);
 +        } finally {
 +          Thread.currentThread().setName(oldThreadName);
 +          runState.set(ScanRunState.FINISHED);
 +        }
 +      }
 +    }
 +
 +    @Override
 +    public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
 +        List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
 +        long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
 +
-       String tableId = new String(textent.getTable(), Constants.UTF8);
++      String tableId = new String(textent.getTable(), UTF_8);
 +      if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      if (!security.userHasAuthorizations(credentials, authorizations))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
 +
 +      KeyExtent extent = new KeyExtent(textent);
 +
 +      // wait for any writes that are in flight.. this done to ensure
 +      // consistency across client restarts... assume a client writes
 +      // to accumulo and dies while waiting for a confirmation from
 +      // accumulo... the client process restarts and tries to read
 +      // data from accumulo making the assumption that it will get
 +      // any writes previously made... however if the server side thread
 +      // processing the write from the dead client is still in progress,
 +      // the restarted client may not see the write unless we wait here.
 +      // this behavior is very important when the client is reading the
 +      // metadata
 +      if (waitForWrites)
 +        writeTracker.waitForWrites(TabletType.type(extent));
 +
 +      Tablet tablet = onlineTablets.get(extent);
 +      if (tablet == null)
 +        throw new NotServingTabletException(textent);
 +
 +      ScanSession scanSession = new ScanSession();
 +      scanSession.user = credentials.getPrincipal();
 +      scanSession.extent = new KeyExtent(extent);
 +      scanSession.columnSet = new HashSet<Column>();
 +      scanSession.ssiList = ssiList;
 +      scanSession.ssio = ssio;
 +      scanSession.auths = new Authorizations(authorizations);
 +      scanSession.interruptFlag = new AtomicBoolean();
 +      scanSession.readaheadThreshold = readaheadThreshold;
 +
 +      for (TColumn tcolumn : columns) {
 +        scanSession.columnSet.add(new Column(tcolumn));
 +      }
 +
 +      scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
 +          scanSession.interruptFlag);
 +
 +      long sid = sessionManager.createSession(scanSession, true);
 +
 +      ScanResult scanResult;
 +      try {
 +        scanResult = continueScan(tinfo, sid, scanSession);
 +      } catch (NoSuchScanIDException e) {
 +        log.error("The impossible happened", e);
 +        throw new RuntimeException();
 +      } finally {
 +        sessionManager.unreserveSession(sid);
 +      }
 +
 +      return new InitialScan(sid, scanResult);
 +    }
 +
 +    @Override
 +    public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
 +        org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
 +      ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
 +      if (scanSession == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      try {
 +        return continueScan(tinfo, scanID, scanSession);
 +      } finally {
 +        sessionManager.unreserveSession(scanSession);
 +      }
 +    }
 +
 +    private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
 +        org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
 +
 +      if (scanSession.nextBatchTask == null) {
 +        scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
 +        resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
 +      }
 +
 +      ScanBatch bresult;
 +      try {
 +        bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
 +        scanSession.nextBatchTask = null;
 +      } catch (ExecutionException e) {
 +        sessionManager.removeSession(scanID);
 +        if (e.getCause() instanceof NotServingTabletException)
 +          throw (NotServingTabletException) e.getCause();
 +        else if (e.getCause() instanceof TooManyFilesException)
 +          throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
 +        else
 +          throw new RuntimeException(e);
 +      } catch (CancellationException ce) {
 +        sessionManager.removeSession(scanID);
 +        Tablet tablet = onlineTablets.get(scanSession.extent);
 +        if (tablet == null || tablet.isClosed())
 +          throw new NotServingTabletException(scanSession.extent.toThrift());
 +        else
 +          throw new NoSuchScanIDException();
 +      } catch (TimeoutException e) {
 +        List<TKeyValue> param = Collections.emptyList();
 +        long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
 +        sessionManager.removeIfNotAccessed(scanID, timeout);
 +        return new ScanResult(param, true);
 +      } catch (Throwable t) {
 +        sessionManager.removeSession(scanID);
 +        log.warn("Failed to get next batch", t);
 +        throw new RuntimeException(t);
 +      }
 +
 +      ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
 +
 +      scanSession.entriesReturned += scanResult.results.size();
 +
 +      scanSession.batchCount++;
 +
 +      if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
 +        // start reading next batch while current batch is transmitted
 +        // to client
 +        scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
 +        resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
 +      }
 +
 +      if (!scanResult.more)
 +        closeScan(tinfo, scanID);
 +
 +      return scanResult;
 +    }
 +
 +    @Override
 +    public void closeScan(TInfo tinfo, long scanID) {
 +      ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
 +      if (ss != null) {
 +        long t2 = System.currentTimeMillis();
 +
 +        log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
 +            .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
 +        if (scanMetrics.isEnabled()) {
 +          scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
 +          scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
 +        }
 +      }
 +    }
 +
 +    @Override
 +    public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
 +        List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
 +      // find all of the tables that need to be scanned
 +      HashSet<String> tables = new HashSet<String>();
 +      for (TKeyExtent keyExtent : tbatch.keySet()) {
-         tables.add(new String(keyExtent.getTable(), Constants.UTF8));
++        tables.add(new String(keyExtent.getTable(), UTF_8));
 +      }
 +
 +      if (tables.size() != 1)
 +        throw new IllegalArgumentException("Cannot batch scan over multiple tables");
 +
 +      // check if user has permission to the tables
 +      for (String tableId : tables)
 +        if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), tbatch, tcolumns, ssiList, ssio, authorizations))
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      try {
 +        if (!security.userHasAuthorizations(credentials, authorizations))
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
 +      } catch (ThriftSecurityException tse) {
 +        log.error(tse, tse);
 +        throw tse;
 +      }
 +      Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, new TKeyExtentTranslator(), new Translator.ListTranslator<TRange,Range>(
 +          new TRangeTranslator()));
 +
 +      // This is used to determine which thread pool to use
 +      KeyExtent threadPoolExtent = batch.keySet().iterator().next();
 +
 +      if (waitForWrites)
 +        writeTracker.waitForWrites(TabletType.type(batch.keySet()));
 +
 +      MultiScanSession mss = new MultiScanSession();
 +      mss.user = credentials.getPrincipal();
 +      mss.queries = batch;
 +      mss.columnSet = new HashSet<Column>(tcolumns.size());
 +      mss.ssiList = ssiList;
 +      mss.ssio = ssio;
 +      mss.auths = new Authorizations(authorizations);
 +
 +      mss.numTablets = batch.size();
 +      for (List<Range> ranges : batch.values()) {
 +        mss.numRanges += ranges.size();
 +      }
 +
 +      for (TColumn tcolumn : tcolumns)
 +        mss.columnSet.add(new Column(tcolumn));
 +
 +      mss.threadPoolExtent = threadPoolExtent;
 +
 +      long sid = sessionManager.createSession(mss, true);
 +
 +      MultiScanResult result;
 +      try {
 +        result = continueMultiScan(tinfo, sid, mss);
 +      } catch (NoSuchScanIDException e) {
 +        log.error("the impossible happened", e);
 +        throw new RuntimeException("the impossible happened", e);
 +      } finally {
 +        sessionManager.unreserveSession(sid);
 +      }
 +
 +      return new InitialMultiScan(sid, result);
 +    }
 +
 +    @Override
 +    public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
 +
 +      MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
 +
 +      if (session == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      try {
 +        return continueMultiScan(tinfo, scanID, session);
 +      } finally {
 +        sessionManager.unreserveSession(session);
 +      }
 +    }
 +
 +    private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
 +
 +      if (session.lookupTask == null) {
 +        session.lookupTask = new LookupTask(scanID);
 +        resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
 +      }
 +
 +      try {
 +        MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
 +        session.lookupTask = null;
 +        return scanResult;
 +      } catch (TimeoutException e1) {
 +        long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
 +        sessionManager.removeIfNotAccessed(scanID, timeout);
 +        List<TKeyValue> results = Collections.emptyList();
 +        Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
 +        List<TKeyExtent> fullScans = Collections.emptyList();
 +        return new MultiScanResult(results, failures, fullScans, null, null, false, true);
 +      } catch (Throwable t) {
 +        sessionManager.removeSession(scanID);
 +        log.warn("Failed to get multiscan result", t);
 +        throw new RuntimeException(t);
 +      }
 +    }
 +
 +    @Override
 +    public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
 +      MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
 +      if (session == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      long t2 = System.currentTimeMillis();
 +      log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
 +          session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
 +    }
 +
 +    @Override
 +    public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
 +      // Make sure user is real
 +
 +      security.authenticateUser(credentials, credentials);
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
 +
 +      UpdateSession us = new UpdateSession();
 +      us.violations = new Violations();
 +      us.credentials = credentials;
 +      us.cenv = new TservConstraintEnv(security, us.credentials);
 +
 +      long sid = sessionManager.createSession(us, false);
 +
 +      return sid;
 +    }
 +
 +    private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
 +      long t1 = System.currentTimeMillis();
 +      if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
 +        return;
 +      if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
 +        // if there were previous failures, then do not accept additional writes
 +        return;
 +      }
 +
 +      try {
 +        // if user has no permission to write to this table, add it to
 +        // the failures list
 +        boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
 +        String tableId = keyExtent.getTableId().toString();
 +        if (sameTable || security.canWrite(us.credentials, tableId, Tables.getNamespaceId(instance, tableId))) {
 +          long t2 = System.currentTimeMillis();
 +          us.authTimes.addStat(t2 - t1);
 +          us.currentTablet = onlineTablets.get(keyExtent);
 +          if (us.currentTablet != null) {
 +            us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
 +          } else {
 +            // not serving tablet, so report all mutations as
 +            // failures
 +            us.failures.put(keyExtent, 0l);
 +            if (updateMetrics.isEnabled())
 +              updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
 +          }
 +        } else {
 +          log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
 +          long t2 = System.currentTimeMillis();
 +          us.authTimes.addStat(t2 - t1);
 +          us.currentTablet = null;
 +          us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
 +          if (updateMetrics.isEnabled())
 +            updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
 +          return;
 +        }
 +      } catch (ThriftSecurityException e) {
 +        log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
 +        long t2 = System.currentTimeMillis();
 +        us.authTimes.addStat(t2 - t1);
 +        us.currentTablet = null;
 +        us.authFailures.put(keyExtent, e.getCode());
 +        if (updateMetrics.isEnabled())
 +          updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
 +        return;
 +      }
 +    }
 +
 +    @Override
 +    public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
 +      UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
 +      if (us == null) {
 +        throw new RuntimeException("No Such SessionID");
 +      }
 +
 +      try {
 +        KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +        setUpdateTablet(us, keyExtent);
 +
 +        if (us.currentTablet != null) {
 +          List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
 +          for (TMutation tmutation : tmutations) {
 +            Mutation mutation = new ServerMutation(tmutation);
 +            mutations.add(mutation);
 +            us.queuedMutationSize += mutation.numBytes();
 +          }
 +          if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
 +            flush(us);
 +        }
 +      } finally {
 +        sessionManager.unreserveSession(us);
 +      }
 +    }
 +
 +    private void flush(UpdateSession us) {
 +
 +      int mutationCount = 0;
 +      Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
 +      Throwable error = null;
 +
 +      long pt1 = System.currentTimeMillis();
 +
 +      boolean containsMetadataTablet = false;
 +      for (Tablet tablet : us.queuedMutations.keySet())
 +        if (tablet.getExtent().isMeta())
 +          containsMetadataTablet = true;
 +
 +      if (!containsMetadataTablet && us.queuedMutations.size() > 0)
 +        TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
 +
 +      Span prep = Trace.start("prep");
 +      try {
 +        for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
 +
 +          Tablet tablet = entry.getKey();
 +          List<Mutation> mutations = entry.getValue();
 +          if (mutations.size() > 0) {
 +            try {
 +              if (updateMetrics.isEnabled())
 +                updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
 +
 +              CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
 +              if (commitSession == null) {
 +                if (us.currentTablet == tablet) {
 +                  us.currentTablet = null;
 +                }
 +                us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
 +              } else {
 +                sendables.put(commitSession, mutations);
 +                mutationCount += mutations.size();
 +              }
 +
 +            } catch (TConstraintViolationException e) {
 +              us.violations.add(e.getViolations());
 +              if (updateMetrics.isEnabled())
 +                updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
 +
 +              if (e.getNonViolators().size() > 0) {
 +                // only log and commit mutations if there were some
 +                // that did not
 +                // violate constraints... this is what
 +                // prepareMutationsForCommit()
 +                // expects
 +                sendables.put(e.getCommitSession(), e.getNonViolators());
 +              }
 +
 +              mutationCount += mutations.size();
 +
 +            } catch (HoldTimeoutException t) {
 +              error = t;
 +              log.debug("Giving up on mutations due to a long memory hold time");
 +              break;
 +            } catch (Throwable t) {
 +              error = t;
 +              log.error("Unexpected error preparing for commit", error);
 +              break;
 +            }
 +          }
 +        }
 +      } finally {
 +        prep.stop();
 +      }
 +
 +      long pt2 = System.currentTimeMillis();
 +      us.prepareTimes.addStat(pt2 - pt1);
 +      updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
 +
 +      if (error != null) {
 +        for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
 +          e.getKey().abortCommit(e.getValue());
 +        }
 +        throw new RuntimeException(error);
 +      }
 +      try {
 +        Span wal = Trace.start("wal");
 +        try {
 +          while (true) {
 +            try {
 +              long t1 = System.currentTimeMillis();
 +
 +              logger.logManyTablets(sendables);
 +
 +              long t2 = System.currentTimeMillis();
 +              us.walogTimes.addStat(t2 - t1);
 +              updateWalogWriteTime((t2 - t1));
 +              break;
 +            } catch (IOException ex) {
 +              log.warn("logging mutations failed, retrying");
 +            } catch (FSError ex) { // happens when DFS is localFS
 +              log.warn("logging mutations failed, retrying");
 +            } catch (Throwable t) {
 +              log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
 +              throw new RuntimeException(t);
 +            }
 +          }
 +        } finally {
 +          wal.stop();
 +        }
 +
 +        Span commit = Trace.start("commit");
 +        try {
 +          long t1 = System.currentTimeMillis();
 +          for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
 +            CommitSession commitSession = entry.getKey();
 +            List<Mutation> mutations = entry.getValue();
 +
 +            commitSession.commit(mutations);
 +
 +            Tablet tablet = commitSession.getTablet();
 +
 +            if (tablet == us.currentTablet) {
 +              // because constraint violations may filter out some
 +              // mutations, for proper
 +              // accounting with the client code, need to increment
 +              // the count based
 +              // on the original number of mutations from the client
 +              // NOT the filtered number
 +              us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
 +            }
 +          }
 +          long t2 = System.currentTimeMillis();
 +
 +          us.flushTime += (t2 - pt1);
 +          us.commitTimes.addStat(t2 - t1);
 +
 +          updateAvgCommitTime(t2 - t1, sendables.size());
 +        } finally {
 +          commit.stop();
 +        }
 +      } finally {
 +        us.queuedMutations.clear();
 +        if (us.currentTablet != null) {
 +          us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
 +        }
 +        us.queuedMutationSize = 0;
 +      }
 +      us.totalUpdates += mutationCount;
 +    }
 +
 +    private void updateWalogWriteTime(long time) {
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, time);
 +    }
 +
 +    private void updateAvgCommitTime(long time, int size) {
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.commitTime, (long) ((time) / (double) size));
 +    }
 +
 +    private void updateAvgPrepTime(long time, int size) {
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (long) ((time) / (double) size));
 +    }
 +
 +    @Override
 +    public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
 +      UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
 +      if (us == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      // clients may or may not see data from an update session while
 +      // it is in progress, however when the update session is closed
 +      // want to ensure that reads wait for the write to finish
 +      long opid = writeTracker.startWrite(us.queuedMutations.keySet());
 +
 +      try {
 +        flush(us);
 +      } finally {
 +        writeTracker.finishWrite(opid);
 +      }
 +
 +      log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
 +          (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
 +          us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
 +      if (us.failures.size() > 0) {
 +        Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
 +        log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
 +      }
 +      List<ConstraintViolationSummary> violations = us.violations.asList();
 +      if (violations.size() > 0) {
 +        ConstraintViolationSummary first = us.violations.asList().iterator().next();
 +        log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
 +      }
 +      if (us.authFailures.size() > 0) {
 +        KeyExtent first = us.authFailures.keySet().iterator().next();
 +        log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
 +      }
 +
 +      return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
 +          us.authFailures, Translators.KET));
 +    }
 +
 +    @Override
 +    public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
 +        ConstraintViolationException, ThriftSecurityException {
 +
-       String tableId = new String(tkeyExtent.getTable(), Constants.UTF8);
++      String tableId = new String(tkeyExtent.getTable(), UTF_8);
 +      if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +      KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +      Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
 +      if (tablet == null) {
 +        throw new NotServingTabletException(tkeyExtent);
 +      }
 +
 +      if (!keyExtent.isMeta())
 +        TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
 +
 +      long opid = writeTracker.startWrite(TabletType.type(keyExtent));
 +
 +      try {
 +        Mutation mutation = new ServerMutation(tmutation);
 +        List<Mutation> mutations = Collections.singletonList(mutation);
 +
 +        Span prep = Trace.start("prep");
 +        CommitSession cs;
 +        try {
 +          cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
 +        } finally {
 +          prep.stop();
 +        }
 +        if (cs == null) {
 +          throw new NotServingTabletException(tkeyExtent);
 +        }
 +
 +        while (true) {
 +          try {
 +            Span wal = Trace.start("wal");
 +            try {
 +              logger.log(cs, cs.getWALogSeq(), mutation);
 +            } finally {
 +              wal.stop();
 +            }
 +            break;
 +          } catch (IOException ex) {
 +            log.warn(ex, ex);
 +          }
 +        }
 +
 +        Span commit = Trace.start("commit");
 +        try {
 +          cs.commit(mutations);
 +        } finally {
 +          commit.stop();
 +        }
 +      } catch (TConstraintViolationException e) {
 +        throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
 +      } finally {
 +        writeTracker.finishWrite(opid);
 +      }
 +    }
 +
 +    private void checkConditions(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs,
 +        List<String> symbols) throws IOException {
 +      Iterator<Entry<KeyExtent,List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
 +
 +      CompressedIterators compressedIters = new CompressedIterators(symbols);
 +
 +      while (iter.hasNext()) {
 +        Entry<KeyExtent,List<ServerConditionalMutation>> entry = iter.next();
 +        Tablet tablet = onlineTablets.get(entry.getKey());
 +
 +        if (tablet == null || tablet.isClosed()) {
 +          for (ServerConditionalMutation scm : entry.getValue())
 +            results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          iter.remove();
 +        } else {
 +          List<ServerConditionalMutation> okMutations = new ArrayList<ServerConditionalMutation>(entry.getValue().size());
 +
 +          for (ServerConditionalMutation scm : entry.getValue()) {
 +            if (checkCondition(results, cs, compressedIters, tablet, scm))
 +              okMutations.add(scm);
 +          }
 +
 +          entry.setValue(okMutations);
 +        }
 +
 +      }
 +    }
 +
 +    boolean checkCondition(ArrayList<TCMResult> results, ConditionalSession cs, CompressedIterators compressedIters, Tablet tablet,
 +        ServerConditionalMutation scm) throws IOException {
 +      boolean add = true;
 +
 +      Set<Column> emptyCols = Collections.emptySet();
 +
 +      for (TCondition tc : scm.getConditions()) {
 +
 +        Range range;
 +        if (tc.hasTimestamp)
 +          range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()), tc.getTs());
 +        else
 +          range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()));
 +
 +        IterConfig ic = compressedIters.decompress(tc.iterators);
 +
 +        Scanner scanner = tablet.createScanner(range, 1, emptyCols, cs.auths, ic.ssiList, ic.ssio, false, cs.interruptFlag);
 +
 +        try {
 +          ScanBatch batch = scanner.read();
 +
 +          Value val = null;
 +
 +          for (KVEntry entry2 : batch.results) {
 +            val = entry2.getValue();
 +            break;
 +          }
 +
 +          if ((val == null ^ tc.getVal() == null) || (val != null && !Arrays.equals(tc.getVal(), val.get()))) {
 +            results.add(new TCMResult(scm.getID(), TCMStatus.REJECTED));
 +            add = false;
 +            break;
 +          }
 +
 +        } catch (TabletClosedException e) {
 +          results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          add = false;
 +          break;
 +        } catch (IterationInterruptedException iie) {
 +          results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          add = false;
 +          break;
 +        } catch (TooManyFilesException tmfe) {
 +          results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          add = false;
 +          break;
 +        }
 +      }
 +      return add;
 +    }
 +
 +    private void writeConditionalMutations(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
 +      Set<Entry<KeyExtent,List<ServerConditionalMutation>>> es = updates.entrySet();
 +
 +      Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
 +
 +      boolean sessionCanceled = sess.interruptFlag.get();
 +
 +      Span prepSpan = Trace.start("prep");
 +      try {
 +        long t1 = System.currentTimeMillis();
 +        for (Entry<KeyExtent,List<ServerConditionalMutation>> entry : es) {
 +          Tablet tablet = onlineTablets.get(entry.getKey());
 +          if (tablet == null || tablet.isClosed() || sessionCanceled) {
 +            for (ServerConditionalMutation scm : entry.getValue())
 +              results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          } else {
 +            try {
 +
 +              @SuppressWarnings("unchecked")
 +              List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
 +              if (mutations.size() > 0) {
 +
 +                CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, sess.credentials), mutations);
 +
 +                if (cs == null) {
 +                  for (ServerConditionalMutation scm : entry.getValue())
 +                    results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +                } else {
 +                  for (ServerConditionalMutation scm : entry.getValue())
 +                    results.add(new TCMResult(scm.getID(), TCMStatus.ACCEPTED));
 +                  sendables.put(cs, mutations);
 +                }
 +              }
 +            } catch (TConstraintViolationException e) {
 +              if (e.getNonViolators().size() > 0) {
 +                sendables.put(e.getCommitSession(), e.getNonViolators());
 +                for (Mutation m : e.getNonViolators())
 +                  results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.ACCEPTED));
 +              }
 +
 +              for (Mutation m : e.getViolators())
 +                results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.VIOLATED));
 +            }
 +          }
 +        }
 +
 +        long t2 = System.currentTimeMillis();
 +        updateAvgPrepTime(t2 - t1, es.size());
 +      } finally {
 +        prepSpan.stop();
 +      }
 +
 +      Span walSpan = Trace.start("wal");
 +      try {
 +        while (true && sendables.size() > 0) {
 +          try {
 +            long t1 = System.currentTimeMillis();
 +            logger.logManyTablets(sendables);
 +            long t2 = System.currentTimeMillis();
 +            updateWalogWriteTime(t2 - t1);
 +            break;
 +          } catch (IOException ex) {
 +            log.warn("logging mutations failed, retrying");
 +          } catch (FSError ex) { // happens when DFS is localFS
 +            log.warn("logging mutations failed, retrying");
 +          } catch (Throwable t) {
 +            log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
 +            throw new RuntimeException(t);
 +          }
 +        }
 +      } finally {
 +        walSpan.stop();
 +      }
 +
 +      Span commitSpan = Trace.start("commit");
 +      try {
 +        long t1 = System.currentTimeMillis();
 +        for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
 +          CommitSession commitSession = entry.getKey();
 +          List<Mutation> mutations = entry.getValue();
 +
 +          commitSession.commit(mutations);
 +        }
 +        long t2 = System.currentTimeMillis();
 +        updateAvgCommitTime(t2 - t1, sendables.size());
 +      } finally {
 +        commitSpan.stop();
 +      }
 +
 +    }
 +
 +    private Map<KeyExtent,List<ServerConditionalMutation>> conditionalUpdate(ConditionalSession cs, Map<KeyExtent,List<ServerConditionalMutation>> updates,
 +        ArrayList<TCMResult> results, List<String> symbols) throws IOException {
 +      // sort each list of mutations, this is done to avoid deadlock and doing seeks in order is more efficient and detect duplicate rows.
 +      ConditionalMutationSet.sortConditionalMutations(updates);
 +
 +      Map<KeyExtent,List<ServerConditionalMutation>> deferred = new HashMap<KeyExtent,List<ServerConditionalMutation>>();
 +
 +      // can not process two mutations for the same row, because one will not see what the other writes
 +      ConditionalMutationSet.deferDuplicatesRows(updates, deferred);
 +
 +      // get as many locks as possible w/o blocking... defer any rows that are locked
 +      List<RowLock> locks = rowLocks.acquireRowlocks(updates, deferred);
 +      try {
 +        Span checkSpan = Trace.start("Check conditions");
 +        try {
 +          checkConditions(updates, results, cs, symbols);
 +        } finally {
 +          checkSpan.stop();
 +        }
 +
 +        Span updateSpan = Trace.start("apply conditional mutations");
 +        try {
 +          writeConditionalMutations(updates, results, cs);
 +        } finally {
 +          updateSpan.stop();
 +        }
 +      } finally {
 +        rowLocks.releaseRowLocks(locks);
 +      }
 +      return deferred;
 +    }
 +
 +    @Override
 +    public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials credentials, List<ByteBuffer> authorizations, String tableId)
 +        throws ThriftSecurityException, TException {
 +
 +      Authorizations userauths = null;
 +      if (!security.canConditionallyUpdate(credentials, tableId, Tables.getNamespaceId(instance, tableId), authorizations))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      userauths = security.getUserAuthorizations(credentials);
 +      for (ByteBuffer auth : authorizations)
 +        if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
 +
 +      ConditionalSession cs = new ConditionalSession();
 +      cs.auths = new Authorizations(authorizations);
 +      cs.credentials = credentials;
 +      cs.tableId = tableId;
 +      cs.interruptFlag = new AtomicBoolean();
 +
 +      long sid = sessionManager.createSession(cs, false);
 +      return new TConditionalSession(sid, lockID, sessionManager.getMaxIdleTime());
 +    }
 +
 +    @Override
 +    public List<TCMResult> conditionalUpdate(TInfo tinfo, long sessID, Map<TKeyExtent,List<TConditionalMutation>> mutations, List<String> symbols)
 +        throws NoSuchScanIDException, TException {
 +
 +      ConditionalSession cs = (ConditionalSession) sessionManager.reserveSession(sessID);
 +
 +      if (cs == null || cs.interruptFlag.get())
 +        throw new NoSuchScanIDException();
 +
 +      if (!cs.tableId.equals(MetadataTable.ID) && !cs.tableId.equals(RootTable.ID))
 +        TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
 +
 +      Text tid = new Text(cs.tableId);
 +      long opid = writeTracker.startWrite(TabletType.type(new KeyExtent(tid, null, null)));
 +
 +      try {
 +        Map<KeyExtent,List<ServerConditionalMutation>> updates = Translator.translate(mutations, Translators.TKET,
 +            new Translator.ListTranslator<TConditionalMutation,ServerConditionalMutation>(ServerConditionalMutation.TCMT));
 +
 +        for (KeyExtent ke : updates.keySet())
 +          if (!ke.getTableId().equals(tid))
 +            throw new IllegalArgumentException("Unexpected table id " + tid + " != " + ke.getTableId());
 +
 +        ArrayList<TCMResult> results = new ArrayList<TCMResult>();
 +
 +        Map<KeyExtent,List<ServerConditionalMutation>> deferred = conditionalUpdate(cs, updates, results, symbols);
 +
 +        while (deferred.size() > 0) {
 +          deferred = conditionalUpdate(cs, deferred, results, symbols);
 +        }
 +
 +        return results;
 +      } catch (IOException ioe) {
 +        throw new TException(ioe);
 +      } finally {
 +        writeTracker.finishWrite(opid);
 +        sessionManager.unreserveSession(sessID);
 +      }
 +    }
 +
 +    @Override
 +    public void invalidateConditionalUpdate(TInfo tinfo, long sessID) throws TException {
 +      // this method should wait for any running conditional update to complete
 +      // after this method returns a conditional update should not be able to start
 +
 +      ConditionalSession cs = (ConditionalSession) sessionManager.getSession(sessID);
 +      if (cs != null)
 +        cs.interruptFlag.set(true);
 +
 +      cs = (ConditionalSession) sessionManager.reserveSession(sessID, true);
 +      if (cs != null)
 +        sessionManager.removeSession(sessID, true);
 +    }
 +
 +    @Override
 +    public void closeConditionalUpdate(TInfo tinfo, long sessID) throws TException {
 +      sessionManager.removeSession(sessID, false);
 +    }
 +
 +    @Override
 +    public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
 +        ThriftSecurityException {
 +
 +      String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table));
 +      String namespaceId;
 +      try {
 +        namespaceId = Tables.getNamespaceId(instance, tableId);
 +      } catch (IllegalArgumentException ex) {
 +        // table does not exist, try to educate the client
 +        throw new NotServingTabletException(tkeyExtent);
 +      }
 +
 +      if (!security.canSplitTablet(credentials, tableId, namespaceId))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +
 +      Tablet tablet = onlineTablets.get(keyExtent);
 +      if (tablet == null) {
 +        throw new NotServingTabletException(tkeyExtent);
 +      }
 +
 +      if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
 +        try {
 +          if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
 +            throw new NotServingTabletException(tkeyExtent);
 +          }
 +        } catch (IOException e) {
 +          log.warn("Failed to split " + keyExtent, e);
 +          throw new RuntimeException(e);
 +        }
 +      }
 +    }
 +
 +    @Override
 +    public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
 +      return getStats(sessionManager.getActiveScansPerTable());
 +    }
 +
 +    @Override
 +    public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
 +      TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
 +      synchronized (onlineTablets) {
 +        onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
 +      }
 +      List<TabletStats> result = new ArrayList<TabletStats>();
 +      Text text = new Text(tableId);
 +      KeyExtent start = new KeyExtent(text, new Text(), null);
 +      for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
 +        KeyExtent ke = entry.getKey();
 +        if (ke.getTableId().compareTo(text) == 0) {
 +          Tablet tablet = entry.getValue();
 +          TabletStats stats = tablet.timer.getTabletStats();
 +          stats.extent = ke.toThrift();
 +          stats.ingestRate = tablet.ingestRate();
 +          stats.queryRate = tablet.queryRate();
 +          stats.splitCreationTime = tablet.getSplitCreationTime();
 +          stats.numEntries = tablet.getNumEntries();
 +          result.add(stats);
 +        }
 +      }
 +      return result;
 +    }
 +
 +    private ZooCache masterLockCache = new ZooCache();
 +
 +    private void checkPermission(TCredentials credentials, String lock, final String request) throws ThriftSecurityException {
 +      boolean fatal = false;
 +      try {
 +        log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
 +        if (!security.canPerformSystemActions(credentials)) {
 +          log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +        }
 +      } catch (ThriftSecurityException e) {
 +        log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
 +        if (SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName())) {
 +          log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configu

<TRUNCATED>

[06/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index 0778f5b,0000000..3459069
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@@ -1,3857 -1,0 +1,3859 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.DataInputStream;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.EnumSet;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.PriorityQueue;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicLong;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.concurrent.locks.ReentrantLock;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Violations;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Column;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.KeyValue;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.MapFileInfo;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVIterator;
 +import org.apache.accumulo.core.iterators.IterationInterruptedException;
 +import org.apache.accumulo.core.iterators.IteratorEnvironment;
 +import org.apache.accumulo.core.iterators.IteratorUtil;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
 +import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
 +import org.apache.accumulo.core.iterators.system.DeletingIterator;
 +import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
 +import org.apache.accumulo.core.iterators.system.MultiIterator;
 +import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
 +import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSource;
 +import org.apache.accumulo.core.iterators.system.StatsIterator;
 +import org.apache.accumulo.core.iterators.system.VisibilityFilter;
 +import org.apache.accumulo.core.master.thrift.TabletLoadState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.LocalityGroupUtil;
 +import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.tableOps.CompactionIterators;
 +import org.apache.accumulo.server.problems.ProblemReport;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.problems.ProblemType;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.FileUtil;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.TabletOperations;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.instrument.Trace;
 +import org.apache.accumulo.tserver.Compactor.CompactionCanceledException;
 +import org.apache.accumulo.tserver.Compactor.CompactionEnv;
 +import org.apache.accumulo.tserver.FileManager.ScanFileManager;
 +import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
 +import org.apache.accumulo.tserver.TabletServer.TservConstraintEnv;
 +import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
 +import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
 +import org.apache.accumulo.tserver.compaction.CompactionPlan;
 +import org.apache.accumulo.tserver.compaction.CompactionStrategy;
 +import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
 +import org.apache.accumulo.tserver.compaction.WriteParameters;
 +import org.apache.accumulo.tserver.constraints.ConstraintChecker;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.MutationReceiver;
 +import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
 +import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
 +import org.apache.commons.codec.DecoderException;
 +import org.apache.commons.codec.binary.Hex;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +import com.google.common.annotations.VisibleForTesting;
 +
 +/*
 + * We need to be able to have the master tell a tabletServer to
 + * close this file, and the tablet server to handle all pending client reads
 + * before closing
 + *
 + */
 +
 +/**
 + *
 + * this class just provides an interface to read from a MapFile mostly takes care of reporting start and end keys
 + *
 + * need this because a single row extent can have multiple columns this manages all the columns (each handled by a store) for a single row-extent
 + *
 + *
 + */
 +
 +public class Tablet {
 +
 +  enum MinorCompactionReason {
 +    USER, SYSTEM, CLOSE, RECOVERY
 +  }
 +
 +  public class CommitSession {
 +
 +    private int seq;
 +    private InMemoryMap memTable;
 +    private int commitsInProgress;
 +    private long maxCommittedTime = Long.MIN_VALUE;
 +
 +    private CommitSession(int seq, InMemoryMap imm) {
 +      this.seq = seq;
 +      this.memTable = imm;
 +      commitsInProgress = 0;
 +    }
 +
 +    public int getWALogSeq() {
 +      return seq;
 +    }
 +
 +    private void decrementCommitsInProgress() {
 +      if (commitsInProgress < 1)
 +        throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
 +
 +      commitsInProgress--;
 +      if (commitsInProgress == 0)
 +        Tablet.this.notifyAll();
 +    }
 +
 +    private void incrementCommitsInProgress() {
 +      if (commitsInProgress < 0)
 +        throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
 +
 +      commitsInProgress++;
 +    }
 +
 +    private void waitForCommitsToFinish() {
 +      while (commitsInProgress > 0) {
 +        try {
 +          Tablet.this.wait(50);
 +        } catch (InterruptedException e) {
 +          log.warn(e, e);
 +        }
 +      }
 +    }
 +
 +    public void abortCommit(List<Mutation> value) {
 +      Tablet.this.abortCommit(this, value);
 +    }
 +
 +    public void commit(List<Mutation> mutations) {
 +      Tablet.this.commit(this, mutations);
 +    }
 +
 +    public Tablet getTablet() {
 +      return Tablet.this;
 +    }
 +
 +    public boolean beginUpdatingLogsUsed(ArrayList<DfsLogger> copy, boolean mincFinish) {
 +      return Tablet.this.beginUpdatingLogsUsed(memTable, copy, mincFinish);
 +    }
 +
 +    public void finishUpdatingLogsUsed() {
 +      Tablet.this.finishUpdatingLogsUsed();
 +    }
 +
 +    public int getLogId() {
 +      return logId;
 +    }
 +
 +    public KeyExtent getExtent() {
 +      return extent;
 +    }
 +
 +    private void updateMaxCommittedTime(long time) {
 +      maxCommittedTime = Math.max(time, maxCommittedTime);
 +    }
 +
 +    private long getMaxCommittedTime() {
 +      if (maxCommittedTime == Long.MIN_VALUE)
 +        throw new IllegalStateException("Tried to read max committed time when it was never set");
 +      return maxCommittedTime;
 +    }
 +
 +  }
 +
 +  private class TabletMemory {
 +    private InMemoryMap memTable;
 +    private InMemoryMap otherMemTable;
 +    private InMemoryMap deletingMemTable;
 +    private int nextSeq = 1;
 +    private CommitSession commitSession;
 +
 +    TabletMemory() {
 +      try {
 +        memTable = new InMemoryMap(acuTableConf);
 +      } catch (LocalityGroupConfigurationError e) {
 +        throw new RuntimeException(e);
 +      }
 +      commitSession = new CommitSession(nextSeq, memTable);
 +      nextSeq += 2;
 +    }
 +
 +    InMemoryMap getMemTable() {
 +      return memTable;
 +    }
 +
 +    InMemoryMap getMinCMemTable() {
 +      return otherMemTable;
 +    }
 +
 +    CommitSession prepareForMinC() {
 +      if (otherMemTable != null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      if (deletingMemTable != null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      otherMemTable = memTable;
 +      try {
 +        memTable = new InMemoryMap(acuTableConf);
 +      } catch (LocalityGroupConfigurationError e) {
 +        throw new RuntimeException(e);
 +      }
 +
 +      CommitSession oldCommitSession = commitSession;
 +      commitSession = new CommitSession(nextSeq, memTable);
 +      nextSeq += 2;
 +
 +      tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), otherMemTable.estimatedSizeInBytes());
 +
 +      return oldCommitSession;
 +    }
 +
 +    void finishedMinC() {
 +
 +      if (otherMemTable == null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      if (deletingMemTable != null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      deletingMemTable = otherMemTable;
 +
 +      otherMemTable = null;
 +      Tablet.this.notifyAll();
 +    }
 +
 +    void finalizeMinC() {
 +      try {
 +        deletingMemTable.delete(15000);
 +      } finally {
 +        synchronized (Tablet.this) {
 +          if (otherMemTable != null) {
 +            throw new IllegalStateException();
 +          }
 +
 +          if (deletingMemTable == null) {
 +            throw new IllegalStateException();
 +          }
 +
 +          deletingMemTable = null;
 +
 +          tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), 0);
 +        }
 +      }
 +    }
 +
 +    boolean memoryReservedForMinC() {
 +      return otherMemTable != null || deletingMemTable != null;
 +    }
 +
 +    void waitForMinC() {
 +      while (otherMemTable != null || deletingMemTable != null) {
 +        try {
 +          Tablet.this.wait(50);
 +        } catch (InterruptedException e) {
 +          log.warn(e, e);
 +        }
 +      }
 +    }
 +
 +    void mutate(CommitSession cm, List<Mutation> mutations) {
 +      cm.memTable.mutate(mutations);
 +    }
 +
 +    void updateMemoryUsageStats() {
 +      long other = 0;
 +      if (otherMemTable != null)
 +        other = otherMemTable.estimatedSizeInBytes();
 +      else if (deletingMemTable != null)
 +        other = deletingMemTable.estimatedSizeInBytes();
 +
 +      tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), other);
 +    }
 +
 +    List<MemoryIterator> getIterators() {
 +      List<MemoryIterator> toReturn = new ArrayList<MemoryIterator>(2);
 +      toReturn.add(memTable.skvIterator());
 +      if (otherMemTable != null)
 +        toReturn.add(otherMemTable.skvIterator());
 +      return toReturn;
 +    }
 +
 +    void returnIterators(List<MemoryIterator> iters) {
 +      for (MemoryIterator iter : iters) {
 +        iter.close();
 +      }
 +    }
 +
 +    public long getNumEntries() {
 +      if (otherMemTable != null)
 +        return memTable.getNumEntries() + otherMemTable.getNumEntries();
 +      return memTable.getNumEntries();
 +    }
 +
 +    CommitSession getCommitSession() {
 +      return commitSession;
 +    }
 +  }
 +
 +  private TabletMemory tabletMemory;
 +
 +  private final TabletTime tabletTime;
 +  private long persistedTime;
 +  private final Object timeLock = new Object();
 +
 +  private final Path location; // absolute path of this tablets dir
 +  private TServerInstance lastLocation;
 +
 +  private Configuration conf;
 +  private VolumeManager fs;
 +
 +  private TableConfiguration acuTableConf;
 +
 +  private volatile boolean tableDirChecked = false;
 +
 +  private AtomicLong dataSourceDeletions = new AtomicLong(0);
 +  private Set<ScanDataSource> activeScans = new HashSet<ScanDataSource>();
 +
 +  private volatile boolean closing = false;
 +  private boolean closed = false;
 +  private boolean closeComplete = false;
 +
 +  private long lastFlushID = -1;
 +  private long lastCompactID = -1;
 +
 +  private static class CompactionWaitInfo {
 +    long flushID = -1;
 +    long compactionID = -1;
 +  }
 +
 +  // stores info about user initiated major compaction that is waiting on a minor compaction to finish
 +  private CompactionWaitInfo compactionWaitInfo = new CompactionWaitInfo();
 +
 +  private KeyExtent extent;
 +
 +  private TabletResourceManager tabletResources;
 +  final private DatafileManager datafileManager;
 +  private volatile boolean majorCompactionInProgress = false;
 +  private volatile boolean majorCompactionWaitingToStart = false;
 +  private Set<MajorCompactionReason> majorCompactionQueued = Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
 +  private volatile boolean minorCompactionInProgress = false;
 +  private volatile boolean minorCompactionWaitingToStart = false;
 +
 +  private boolean updatingFlushID = false;
 +
 +  private AtomicReference<ConstraintChecker> constraintChecker = new AtomicReference<ConstraintChecker>();
 +
 +  private final String tabletDirectory;
 +
 +  private int writesInProgress = 0;
 +
 +  private static final Logger log = Logger.getLogger(Tablet.class);
 +  public TabletStatsKeeper timer;
 +
 +  private Rate queryRate = new Rate(0.2);
 +  private long queryCount = 0;
 +
 +  private Rate queryByteRate = new Rate(0.2);
 +  private long queryBytes = 0;
 +
 +  private Rate ingestRate = new Rate(0.2);
 +  private long ingestCount = 0;
 +
 +  private Rate ingestByteRate = new Rate(0.2);
 +  private long ingestBytes = 0;
 +
 +  private byte[] defaultSecurityLabel = new byte[0];
 +
 +  private long lastMinorCompactionFinishTime;
 +  private long lastMapFileImportTime;
 +
 +  private volatile long numEntries;
 +  private volatile long numEntriesInMemory;
 +
 +  // a count of the amount of data read by the iterators
 +  private AtomicLong scannedCount = new AtomicLong(0);
 +  private Rate scannedRate = new Rate(0.2);
 +
 +  private ConfigurationObserver configObserver;
 +
 +  private TabletServer tabletServer;
 +
 +  private final int logId;
 +  // ensure we only have one reader/writer of our bulk file notes at at time
 +  public final Object bulkFileImportLock = new Object();
 +
 +  public int getLogId() {
 +    return logId;
 +  }
 +
 +  public static class TabletClosedException extends RuntimeException {
 +    public TabletClosedException(Exception e) {
 +      super(e);
 +    }
 +
 +    public TabletClosedException() {
 +      super();
 +    }
 +
 +    private static final long serialVersionUID = 1L;
 +  }
 +
 +  FileRef getNextMapFilename(String prefix) throws IOException {
 +    String extension = FileOperations.getNewFileExtension(tabletServer.getTableConfiguration(extent));
 +    checkTabletDir();
 +    return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension);
 +  }
 +
 +  private void checkTabletDir() throws IOException {
 +    if (!tableDirChecked) {
 +      checkTabletDir(this.location);
 +      tableDirChecked = true;
 +    }
 +  }
 +
 +  private void checkTabletDir(Path tabletDir) throws IOException {
 +
 +    FileStatus[] files = null;
 +    try {
 +      files = fs.listStatus(tabletDir);
 +    } catch (FileNotFoundException ex) {
 +      // ignored
 +    }
 +
 +    if (files == null) {
 +      if (tabletDir.getName().startsWith(Constants.CLONE_PREFIX))
 +        log.debug("Tablet " + extent + " had no dir, creating " + tabletDir); // its a clone dir...
 +      else
 +        log.warn("Tablet " + extent + " had no dir, creating " + tabletDir);
 +
 +      fs.mkdirs(tabletDir);
 +    }
 +  }
 +
 +  class DatafileManager {
 +    // access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
 +    final private Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
 +
 +    DatafileManager(SortedMap<FileRef,DataFileValue> datafileSizes) {
 +      for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet())
 +        this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
 +    }
 +
 +    FileRef mergingMinorCompactionFile = null;
 +    Set<FileRef> filesToDeleteAfterScan = new HashSet<FileRef>();
 +    Map<Long,Set<FileRef>> scanFileReservations = new HashMap<Long,Set<FileRef>>();
 +    MapCounter<FileRef> fileScanReferenceCounts = new MapCounter<FileRef>();
 +    long nextScanReservationId = 0;
 +    boolean reservationsBlocked = false;
 +
 +    Set<FileRef> majorCompactingFiles = new HashSet<FileRef>();
 +
 +    Pair<Long,Map<FileRef,DataFileValue>> reserveFilesForScan() {
 +      synchronized (Tablet.this) {
 +
 +        while (reservationsBlocked) {
 +          try {
 +            Tablet.this.wait(50);
 +          } catch (InterruptedException e) {
 +            log.warn(e, e);
 +          }
 +        }
 +
 +        Set<FileRef> absFilePaths = new HashSet<FileRef>(datafileSizes.keySet());
 +
 +        long rid = nextScanReservationId++;
 +
 +        scanFileReservations.put(rid, absFilePaths);
 +
 +        Map<FileRef,DataFileValue> ret = new HashMap<FileRef,DataFileValue>();
 +
 +        for (FileRef path : absFilePaths) {
 +          fileScanReferenceCounts.increment(path, 1);
 +          ret.put(path, datafileSizes.get(path));
 +        }
 +
 +        return new Pair<Long,Map<FileRef,DataFileValue>>(rid, ret);
 +      }
 +    }
 +
 +    void returnFilesForScan(Long reservationId) {
 +
 +      final Set<FileRef> filesToDelete = new HashSet<FileRef>();
 +
 +      synchronized (Tablet.this) {
 +        Set<FileRef> absFilePaths = scanFileReservations.remove(reservationId);
 +
 +        if (absFilePaths == null)
 +          throw new IllegalArgumentException("Unknown scan reservation id " + reservationId);
 +
 +        boolean notify = false;
 +        for (FileRef path : absFilePaths) {
 +          long refCount = fileScanReferenceCounts.decrement(path, 1);
 +          if (refCount == 0) {
 +            if (filesToDeleteAfterScan.remove(path))
 +              filesToDelete.add(path);
 +            notify = true;
 +          } else if (refCount < 0)
 +            throw new IllegalStateException("Scan ref count for " + path + " is " + refCount);
 +        }
 +
 +        if (notify)
 +          Tablet.this.notifyAll();
 +      }
 +
 +      if (filesToDelete.size() > 0) {
 +        log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
 +        MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
 +      }
 +    }
 +
 +    private void removeFilesAfterScan(Set<FileRef> scanFiles) {
 +      if (scanFiles.size() == 0)
 +        return;
 +
 +      Set<FileRef> filesToDelete = new HashSet<FileRef>();
 +
 +      synchronized (Tablet.this) {
 +        for (FileRef path : scanFiles) {
 +          if (fileScanReferenceCounts.get(path) == 0)
 +            filesToDelete.add(path);
 +          else
 +            filesToDeleteAfterScan.add(path);
 +        }
 +      }
 +
 +      if (filesToDelete.size() > 0) {
 +        log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
 +        MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
 +      }
 +    }
 +
 +    private TreeSet<FileRef> waitForScansToFinish(Set<FileRef> pathsToWaitFor, boolean blockNewScans, long maxWaitTime) {
 +      long startTime = System.currentTimeMillis();
 +      TreeSet<FileRef> inUse = new TreeSet<FileRef>();
 +
 +      Span waitForScans = Trace.start("waitForScans");
 +      try {
 +        synchronized (Tablet.this) {
 +          if (blockNewScans) {
 +            if (reservationsBlocked)
 +              throw new IllegalStateException();
 +
 +            reservationsBlocked = true;
 +          }
 +
 +          for (FileRef path : pathsToWaitFor) {
 +            while (fileScanReferenceCounts.get(path) > 0 && System.currentTimeMillis() - startTime < maxWaitTime) {
 +              try {
 +                Tablet.this.wait(100);
 +              } catch (InterruptedException e) {
 +                log.warn(e, e);
 +              }
 +            }
 +          }
 +
 +          for (FileRef path : pathsToWaitFor) {
 +            if (fileScanReferenceCounts.get(path) > 0)
 +              inUse.add(path);
 +          }
 +
 +          if (blockNewScans) {
 +            reservationsBlocked = false;
 +            Tablet.this.notifyAll();
 +          }
 +
 +        }
 +      } finally {
 +        waitForScans.stop();
 +      }
 +      return inUse;
 +    }
 +
 +    public void importMapFiles(long tid, Map<FileRef,DataFileValue> pathsString, boolean setTime) throws IOException {
 +
 +      String bulkDir = null;
 +
 +      Map<FileRef,DataFileValue> paths = new HashMap<FileRef,DataFileValue>();
 +      for (Entry<FileRef,DataFileValue> entry : pathsString.entrySet())
 +        paths.put(entry.getKey(), entry.getValue());
 +
 +      for (FileRef tpath : paths.keySet()) {
 +
 +        boolean inTheRightDirectory = false;
 +        Path parent = tpath.path().getParent().getParent();
 +        for (String tablesDir : ServerConstants.getTablesDirs()) {
 +          if (parent.equals(new Path(tablesDir, extent.getTableId().toString()))) {
 +            inTheRightDirectory = true;
 +            break;
 +          }
 +        }
 +        if (!inTheRightDirectory) {
 +          throw new IOException("Data file " + tpath + " not in table dirs");
 +        }
 +
 +        if (bulkDir == null)
 +          bulkDir = tpath.path().getParent().toString();
 +        else if (!bulkDir.equals(tpath.path().getParent().toString()))
 +          throw new IllegalArgumentException("bulk files in different dirs " + bulkDir + " " + tpath);
 +
 +      }
 +
 +      if (extent.isRootTablet()) {
 +        throw new IllegalArgumentException("Can not import files to root tablet");
 +      }
 +
 +      synchronized (bulkFileImportLock) {
 +        Credentials creds = SystemCredentials.get();
 +        Connector conn;
 +        try {
 +          conn = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
 +        } catch (Exception ex) {
 +          throw new IOException(ex);
 +        }
 +        // Remove any bulk files we've previously loaded and compacted away
 +        List<FileRef> files = MetadataTableUtil.getBulkFilesLoaded(conn, extent, tid);
 +
 +        for (FileRef file : files)
 +          if (paths.keySet().remove(file))
 +            log.debug("Ignoring request to re-import a file already imported: " + extent + ": " + file);
 +
 +        if (paths.size() > 0) {
 +          long bulkTime = Long.MIN_VALUE;
 +          if (setTime) {
 +            for (DataFileValue dfv : paths.values()) {
 +              long nextTime = tabletTime.getAndUpdateTime();
 +              if (nextTime < bulkTime)
 +                throw new IllegalStateException("Time went backwards unexpectedly " + nextTime + " " + bulkTime);
 +              bulkTime = nextTime;
 +              dfv.setTime(bulkTime);
 +            }
 +          }
 +
 +          synchronized (timeLock) {
 +            if (bulkTime > persistedTime)
 +              persistedTime = bulkTime;
 +
 +            MetadataTableUtil.updateTabletDataFile(tid, extent, paths, tabletTime.getMetadataValue(persistedTime), creds, tabletServer.getLock());
 +          }
 +        }
 +      }
 +
 +      synchronized (Tablet.this) {
 +        for (Entry<FileRef,DataFileValue> tpath : paths.entrySet()) {
 +          if (datafileSizes.containsKey(tpath.getKey())) {
 +            log.error("Adding file that is already in set " + tpath.getKey());
 +          }
 +          datafileSizes.put(tpath.getKey(), tpath.getValue());
 +
 +        }
 +
 +        tabletResources.importedMapFiles();
 +
 +        computeNumEntries();
 +      }
 +
 +      for (Entry<FileRef,DataFileValue> entry : paths.entrySet()) {
 +        log.log(TLevel.TABLET_HIST, extent + " import " + entry.getKey() + " " + entry.getValue());
 +      }
 +    }
 +
 +    FileRef reserveMergingMinorCompactionFile() {
 +      if (mergingMinorCompactionFile != null)
 +        throw new IllegalStateException("Tried to reserve merging minor compaction file when already reserved  : " + mergingMinorCompactionFile);
 +
 +      if (extent.isRootTablet())
 +        return null;
 +
 +      int maxFiles = acuTableConf.getMaxFilesPerTablet();
 +
 +      // when a major compaction is running and we are at max files, write out
 +      // one extra file... want to avoid the case where major compaction is
 +      // compacting everything except for the largest file, and therefore the
 +      // largest file is returned for merging.. the following check mostly
 +      // avoids this case, except for the case where major compactions fail or
 +      // are canceled
 +      if (majorCompactingFiles.size() > 0 && datafileSizes.size() == maxFiles)
 +        return null;
 +
 +      if (datafileSizes.size() >= maxFiles) {
 +        // find the smallest file
 +
 +        long min = Long.MAX_VALUE;
 +        FileRef minName = null;
 +
 +        for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
 +          if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey())) {
 +            min = entry.getValue().getSize();
 +            minName = entry.getKey();
 +          }
 +        }
 +
 +        if (minName == null)
 +          return null;
 +
 +        mergingMinorCompactionFile = minName;
 +        return minName;
 +      }
 +
 +      return null;
 +    }
 +
 +    void unreserveMergingMinorCompactionFile(FileRef file) {
 +      if ((file == null && mergingMinorCompactionFile != null) || (file != null && mergingMinorCompactionFile == null)
 +          || (file != null && mergingMinorCompactionFile != null && !file.equals(mergingMinorCompactionFile)))
 +        throw new IllegalStateException("Disagreement " + file + " " + mergingMinorCompactionFile);
 +
 +      mergingMinorCompactionFile = null;
 +    }
 +
 +    void bringMinorCompactionOnline(FileRef tmpDatafile, FileRef newDatafile, FileRef absMergeFile, DataFileValue dfv, CommitSession commitSession, long flushId)
 +        throws IOException {
 +
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      if (extent.isRootTablet()) {
 +        try {
 +          if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
 +            throw new IllegalStateException();
 +          }
 +        } catch (Exception e) {
 +          throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
 +        }
 +      }
 +
 +      // rename before putting in metadata table, so files in metadata table should
 +      // always exist
 +      do {
 +        try {
 +          if (dfv.getNumEntries() == 0) {
 +            fs.deleteRecursively(tmpDatafile.path());
 +          } else {
 +            if (fs.exists(newDatafile.path())) {
 +              log.warn("Target map file already exist " + newDatafile);
 +              fs.deleteRecursively(newDatafile.path());
 +            }
 +
 +            rename(fs, tmpDatafile.path(), newDatafile.path());
 +          }
 +          break;
 +        } catch (IOException ioe) {
 +          log.warn("Tablet " + extent + " failed to rename " + newDatafile + " after MinC, will retry in 60 secs...", ioe);
 +          UtilWaitThread.sleep(60 * 1000);
 +        }
 +      } while (true);
 +
 +      long t1, t2;
 +
 +      // the code below always assumes merged files are in use by scans... this must be done
 +      // because the in memory list of files is not updated until after the metadata table
 +      // therefore the file is available to scans until memory is updated, but want to ensure
 +      // the file is not available for garbage collection... if memory were updated
 +      // before this point (like major compactions do), then the following code could wait
 +      // for scans to finish like major compactions do.... used to wait for scans to finish
 +      // here, but that was incorrect because a scan could start after waiting but before
 +      // memory was updated... assuming the file is always in use by scans leads to
 +      // one uneeded metadata update when it was not actually in use
 +      Set<FileRef> filesInUseByScans = Collections.emptySet();
 +      if (absMergeFile != null)
 +        filesInUseByScans = Collections.singleton(absMergeFile);
 +
 +      // very important to write delete entries outside of log lock, because
 +      // this metadata write does not go up... it goes sideways or to itself
 +      if (absMergeFile != null)
 +        MetadataTableUtil.addDeleteEntries(extent, Collections.singleton(absMergeFile), SystemCredentials.get());
 +
 +      Set<String> unusedWalLogs = beginClearingUnusedLogs();
 +      try {
 +        // the order of writing to metadata and walog is important in the face of machine/process failures
 +        // need to write to metadata before writing to walog, when things are done in the reverse order
 +        // data could be lost... the minor compaction start even should be written before the following metadata
 +        // write is made
 +
 +        synchronized (timeLock) {
 +          if (commitSession.getMaxCommittedTime() > persistedTime)
 +            persistedTime = commitSession.getMaxCommittedTime();
 +
 +          String time = tabletTime.getMetadataValue(persistedTime);
 +          MasterMetadataUtil.updateTabletDataFile(extent, newDatafile, absMergeFile, dfv, time, SystemCredentials.get(), filesInUseByScans,
 +              tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, lastLocation, flushId);
 +        }
 +
 +      } finally {
 +        finishClearingUnusedLogs();
 +      }
 +
 +      do {
 +        try {
 +          // the purpose of making this update use the new commit session, instead of the old one passed in,
 +          // is because the new one will reference the logs used by current memory...
 +
 +          tabletServer.minorCompactionFinished(tabletMemory.getCommitSession(), newDatafile.toString(), commitSession.getWALogSeq() + 2);
 +          break;
 +        } catch (IOException e) {
 +          log.error("Failed to write to write-ahead log " + e.getMessage() + " will retry", e);
 +          UtilWaitThread.sleep(1 * 1000);
 +        }
 +      } while (true);
 +
 +      synchronized (Tablet.this) {
 +        lastLocation = null;
 +
 +        t1 = System.currentTimeMillis();
 +        if (datafileSizes.containsKey(newDatafile)) {
 +          log.error("Adding file that is already in set " + newDatafile);
 +        }
 +
 +        if (dfv.getNumEntries() > 0) {
 +          datafileSizes.put(newDatafile, dfv);
 +        }
 +
 +        if (absMergeFile != null) {
 +          datafileSizes.remove(absMergeFile);
 +        }
 +
 +        unreserveMergingMinorCompactionFile(absMergeFile);
 +
 +        dataSourceDeletions.incrementAndGet();
 +        tabletMemory.finishedMinC();
 +
 +        lastFlushID = flushId;
 +
 +        computeNumEntries();
 +        t2 = System.currentTimeMillis();
 +      }
 +
 +      // must do this after list of files in memory is updated above
 +      removeFilesAfterScan(filesInUseByScans);
 +
 +      if (absMergeFile != null)
 +        log.log(TLevel.TABLET_HIST, extent + " MinC [" + absMergeFile + ",memory] -> " + newDatafile);
 +      else
 +        log.log(TLevel.TABLET_HIST, extent + " MinC [memory] -> " + newDatafile);
 +      log.debug(String.format("MinC finish lock %.2f secs %s", (t2 - t1) / 1000.0, getExtent().toString()));
 +      if (dfv.getSize() > acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD)) {
 +        log.debug(String.format("Minor Compaction wrote out file larger than split threshold.  split threshold = %,d  file size = %,d",
 +            acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD), dfv.getSize()));
 +      }
 +
 +    }
 +
 +    public void reserveMajorCompactingFiles(Collection<FileRef> files) {
 +      if (majorCompactingFiles.size() != 0)
 +        throw new IllegalStateException("Major compacting files not empty " + majorCompactingFiles);
 +
 +      if (mergingMinorCompactionFile != null && files.contains(mergingMinorCompactionFile))
 +        throw new IllegalStateException("Major compaction tried to resrve file in use by minor compaction " + mergingMinorCompactionFile);
 +
 +      majorCompactingFiles.addAll(files);
 +    }
 +
 +    public void clearMajorCompactingFile() {
 +      majorCompactingFiles.clear();
 +    }
 +
 +    void bringMajorCompactionOnline(Set<FileRef> oldDatafiles, FileRef tmpDatafile, FileRef newDatafile, Long compactionId, DataFileValue dfv)
 +        throws IOException {
 +      long t1, t2;
 +
 +      if (!extent.isRootTablet()) {
 +
 +        if (fs.exists(newDatafile.path())) {
 +          log.error("Target map file already exist " + newDatafile, new Exception());
 +          throw new IllegalStateException("Target map file already exist " + newDatafile);
 +        }
 +
 +        // rename before putting in metadata table, so files in metadata table should
 +        // always exist
 +        rename(fs, tmpDatafile.path(), newDatafile.path());
 +
 +        if (dfv.getNumEntries() == 0) {
 +          fs.deleteRecursively(newDatafile.path());
 +        }
 +      }
 +
 +      TServerInstance lastLocation = null;
 +      synchronized (Tablet.this) {
 +
 +        t1 = System.currentTimeMillis();
 +
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +        dataSourceDeletions.incrementAndGet();
 +
 +        if (extent.isRootTablet()) {
 +
 +          waitForScansToFinish(oldDatafiles, true, Long.MAX_VALUE);
 +
 +          try {
 +            if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
 +              throw new IllegalStateException();
 +            }
 +          } catch (Exception e) {
 +            throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
 +          }
 +
 +          // mark files as ready for deletion, but
 +          // do not delete them until we successfully
 +          // rename the compacted map file, in case
 +          // the system goes down
 +
 +          RootFiles.replaceFiles(acuTableConf, fs, location, oldDatafiles, tmpDatafile, newDatafile);
 +        }
 +
 +        // atomically remove old files and add new file
 +        for (FileRef oldDatafile : oldDatafiles) {
 +          if (!datafileSizes.containsKey(oldDatafile)) {
 +            log.error("file does not exist in set " + oldDatafile);
 +          }
 +          datafileSizes.remove(oldDatafile);
 +          majorCompactingFiles.remove(oldDatafile);
 +        }
 +
 +        if (datafileSizes.containsKey(newDatafile)) {
 +          log.error("Adding file that is already in set " + newDatafile);
 +        }
 +
 +        if (dfv.getNumEntries() > 0) {
 +          datafileSizes.put(newDatafile, dfv);
 +        }
 +
 +        // could be used by a follow on compaction in a multipass compaction
 +        majorCompactingFiles.add(newDatafile);
 +
 +        computeNumEntries();
 +
 +        lastLocation = Tablet.this.lastLocation;
 +        Tablet.this.lastLocation = null;
 +
 +        if (compactionId != null)
 +          lastCompactID = compactionId;
 +
 +        t2 = System.currentTimeMillis();
 +      }
 +
 +      if (!extent.isRootTablet()) {
 +        Set<FileRef> filesInUseByScans = waitForScansToFinish(oldDatafiles, false, 10000);
 +        if (filesInUseByScans.size() > 0)
 +          log.debug("Adding scan refs to metadata " + extent + " " + filesInUseByScans);
 +        MasterMetadataUtil.replaceDatafiles(extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, SystemCredentials.get(),
 +            tabletServer.getClientAddressString(), lastLocation, tabletServer.getLock());
 +        removeFilesAfterScan(filesInUseByScans);
 +      }
 +
 +      log.debug(String.format("MajC finish lock %.2f secs", (t2 - t1) / 1000.0));
 +      log.log(TLevel.TABLET_HIST, extent + " MajC " + oldDatafiles + " --> " + newDatafile);
 +    }
 +
 +    public SortedMap<FileRef,DataFileValue> getDatafileSizes() {
 +      synchronized (Tablet.this) {
 +        TreeMap<FileRef,DataFileValue> copy = new TreeMap<FileRef,DataFileValue>(datafileSizes);
 +        return Collections.unmodifiableSortedMap(copy);
 +      }
 +    }
 +
 +    public Set<FileRef> getFiles() {
 +      synchronized (Tablet.this) {
 +        HashSet<FileRef> files = new HashSet<FileRef>(datafileSizes.keySet());
 +        return Collections.unmodifiableSet(files);
 +      }
 +    }
 +
 +  }
 +
 +  public Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
 +      throws IOException {
 +    this(tabletServer, location, extent, trm, CachedConfiguration.getInstance(), tabletsKeyValues);
 +    splitCreationTime = 0;
 +  }
 +
 +  public Tablet(KeyExtent extent, TabletServer tabletServer, TabletResourceManager trm, SplitInfo info) throws IOException {
 +    this(tabletServer, new Text(info.dir), extent, trm, CachedConfiguration.getInstance(), info.datafiles, info.time, info.initFlushID, info.initCompactID, info.lastLocation);
 +    splitCreationTime = System.currentTimeMillis();
 +  }
 +
 +  /**
 +   * Only visibile for testing
 +   */
 +  @VisibleForTesting
 +  protected Tablet(TabletTime tabletTime, String tabletDirectory, int logId, Path location, DatafileManager datafileManager) {
 +    this.tabletTime = tabletTime;
 +    this.tabletDirectory = tabletDirectory;
 +    this.logId = logId;
 +    this.location = location;
 +    this.datafileManager = datafileManager;
 +  }
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +    this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), tabletsKeyValues);
 +  }
 +
 +  static private final List<LogEntry> EMPTY = Collections.emptyList();
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
 +      SortedMap<FileRef,DataFileValue> datafiles, String time, long initFlushID, long initCompactID, TServerInstance last) throws IOException {
 +    this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), EMPTY, datafiles, time, last, new HashSet<FileRef>(), initFlushID, initCompactID);
 +  }
 +
 +  private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    SortedMap<Key,Value> entries;
 +
 +    if (extent.isRootTablet()) {
 +      return null;
 +    } else {
 +      entries = new TreeMap<Key,Value>();
 +      Text rowName = extent.getMetadataEntry();
 +      for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +        if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
 +          entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
 +        }
 +      }
 +    }
 +
 +    // log.debug("extent : "+extent+"   entries : "+entries);
 +
 +    if (entries.size() == 1)
 +      return entries.values().iterator().next().toString();
 +    return null;
 +  }
 +
 +  private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +
 +    TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
 +
 +    if (extent.isRootTablet()) { // the meta0 tablet
 +      Path location = new Path(MetadataTableUtil.getRootTabletDir());
 +
 +      // cleanUpFiles() has special handling for delete. files
 +      FileStatus[] files = fs.listStatus(location);
 +      Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
 +      for (String good : goodPaths) {
 +        Path path = new Path(good);
 +        String filename = path.getName();
 +        FileRef ref = new FileRef(location.toString() + "/" + filename, path);
 +        DataFileValue dfv = new DataFileValue(0, 0);
 +        datafiles.put(ref, dfv);
 +      }
 +    } else {
 +
 +      Text rowName = extent.getMetadataEntry();
 +
 +      String tableId = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
 +      ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), tableId, Authorizations.EMPTY);
 +
 +      // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
 +      // reduced batch size to improve performance
 +      // changed here after endKeys were implemented from 10 to 1000
 +      mdScanner.setBatchSize(1000);
 +
 +      // leave these in, again, now using endKey for safety
 +      mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +
 +      mdScanner.setRange(new Range(rowName));
 +
 +      for (Entry<Key,Value> entry : mdScanner) {
 +
 +        if (entry.getKey().compareRow(rowName) != 0) {
 +          break;
 +        }
 +
 +        FileRef ref = new FileRef(fs, entry.getKey());
 +        datafiles.put(ref, new DataFileValue(entry.getValue().get()));
 +      }
 +    }
 +    return datafiles;
 +  }
 +
 +  private static List<LogEntry> lookupLogEntries(KeyExtent ke, SortedMap<Key,Value> tabletsKeyValues) {
 +    List<LogEntry> logEntries = new ArrayList<LogEntry>();
 +
 +    if (ke.isMeta()) {
 +      try {
 +        logEntries = MetadataTableUtil.getLogEntries(SystemCredentials.get(), ke);
 +      } catch (Exception ex) {
 +        throw new RuntimeException("Unable to read tablet log entries", ex);
 +      }
 +    } else {
 +      log.debug("Looking at metadata " + tabletsKeyValues);
 +      Text row = ke.getMetadataEntry();
 +      for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +        Key key = entry.getKey();
 +        if (key.getRow().equals(row)) {
 +          if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
 +            logEntries.add(LogEntry.fromKeyValue(key, entry.getValue()));
 +          }
 +        }
 +      }
 +    }
 +
 +    log.debug("got " + logEntries + " for logs for " + ke);
 +    return logEntries;
 +  }
 +
 +  private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
 +    HashSet<FileRef> scanFiles = new HashSet<FileRef>();
 +
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
 +        scanFiles.add(new FileRef(fs, key));
 +      }
 +    }
 +
 +    return scanFiles;
 +  }
 +
 +  private static long lookupFlushID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +        return Long.parseLong(entry.getValue().toString());
 +    }
 +
 +    return -1;
 +  }
 +
 +  private static long lookupCompactID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +        return Long.parseLong(entry.getValue().toString());
 +    }
 +
 +    return -1;
 +  }
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, VolumeManager fs,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +    this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(), fs,
 +        extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent, tabletsKeyValues),
 +        lookupScanFiles(extent, tabletsKeyValues, fs), lookupFlushID(extent, tabletsKeyValues), lookupCompactID(extent, tabletsKeyValues));
 +  }
 +
 +  private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
 +        return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
 +      }
 +    }
 +    return null;
 +  }
 +
 +  /**
 +   * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
 +   */
 +  private Tablet(final TabletServer tabletServer, final Text location, final KeyExtent extent, final TabletResourceManager trm, final Configuration conf,
 +      final VolumeManager fs, final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time,
 +      final TServerInstance lastLocation, Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
 +
 +    TabletFiles tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getLock(), fs, extent, new TabletFiles(location.toString(), rawLogEntries,
 +        rawDatafiles));
 +
 +    Path locationPath;
 +
 +    if (tabletPaths.dir.contains(":")) {
 +      locationPath = new Path(tabletPaths.dir.toString());
 +    } else {
 +      locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString());
 +    }
 +
 +    final List<LogEntry> logEntries = tabletPaths.logEntries;
 +    final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
 +
 +    this.location = locationPath;
 +    this.lastLocation = lastLocation;
 +    this.tabletDirectory = tabletPaths.dir;
 +    this.conf = conf;
 +    this.acuTableConf = tabletServer.getTableConfiguration(extent);
 +
 +    this.fs = fs;
 +    this.extent = extent;
 +    this.tabletResources = trm;
 +
 +    this.lastFlushID = initFlushID;
 +    this.lastCompactID = initCompactID;
 +
 +    if (extent.isRootTablet()) {
 +      long rtime = Long.MIN_VALUE;
 +      for (FileRef ref : datafiles.keySet()) {
 +        Path path = ref.path();
 +        FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
 +        FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
 +        long maxTime = -1;
 +        try {
 +
 +          while (reader.hasTop()) {
 +            maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
 +            reader.next();
 +          }
 +
 +        } finally {
 +          reader.close();
 +        }
 +
 +        if (maxTime > rtime) {
 +          time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
 +          rtime = maxTime;
 +        }
 +      }
 +    }
 +    if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
 +      // recovery... old root tablet has no data, so time doesn't matter:
 +      time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
 +    }
 +
 +    this.tabletServer = tabletServer;
 +    this.logId = tabletServer.createLogId(extent);
 +
 +    this.timer = new TabletStatsKeeper();
 +
 +    setupDefaultSecurityLabels(extent);
 +
 +    tabletMemory = new TabletMemory();
 +    tabletTime = TabletTime.getInstance(time);
 +    persistedTime = tabletTime.getTime();
 +
 +    acuTableConf.addObserver(configObserver = new ConfigurationObserver() {
 +
 +      private void reloadConstraints() {
 +        constraintChecker.set(new ConstraintChecker(acuTableConf));
 +      }
 +
 +      @Override
 +      public void propertiesChanged() {
 +        reloadConstraints();
 +
 +        try {
 +          setupDefaultSecurityLabels(extent);
 +        } catch (Exception e) {
 +          log.error("Failed to reload default security labels for extent: " + extent.toString());
 +        }
 +      }
 +
 +      @Override
 +      public void propertyChanged(String prop) {
 +        if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
 +          reloadConstraints();
 +        else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
 +          try {
 +            log.info("Default security labels changed for extent: " + extent.toString());
 +            setupDefaultSecurityLabels(extent);
 +          } catch (Exception e) {
 +            log.error("Failed to reload default security labels for extent: " + extent.toString());
 +          }
 +        }
 +
 +      }
 +
 +      @Override
 +      public void sessionExpired() {
 +        log.debug("Session expired, no longer updating per table props...");
 +      }
 +
 +    });
 +
 +    acuTableConf.getNamespaceConfiguration().addObserver(configObserver);
 +
 +    // Force a load of any per-table properties
 +    configObserver.propertiesChanged();
 +
 +    tabletResources.setTablet(this, acuTableConf);
 +    if (!logEntries.isEmpty()) {
 +      log.info("Starting Write-Ahead Log recovery for " + this.extent);
 +      // count[0] = entries used on tablet
 +      // count[1] = track max time from walog entries wihtout timestamps
 +      final long[] count = new long[2];
 +      final CommitSession commitSession = tabletMemory.getCommitSession();
 +      count[1] = Long.MIN_VALUE;
 +      try {
 +        Set<String> absPaths = new HashSet<String>();
 +        for (FileRef ref : datafiles.keySet())
 +          absPaths.add(ref.path().toString());
 +
 +        tabletServer.recover(this.tabletServer.getFileSystem(), this, logEntries, absPaths, new MutationReceiver() {
 +          @Override
 +          public void receive(Mutation m) {
 +            // LogReader.printMutation(m);
 +            Collection<ColumnUpdate> muts = m.getUpdates();
 +            for (ColumnUpdate columnUpdate : muts) {
 +              if (!columnUpdate.hasTimestamp()) {
 +                // if it is not a user set timestamp, it must have been set
 +                // by the system
 +                count[1] = Math.max(count[1], columnUpdate.getTimestamp());
 +              }
 +            }
 +            tabletMemory.mutate(commitSession, Collections.singletonList(m));
 +            count[0]++;
 +          }
 +        });
 +
 +        if (count[1] != Long.MIN_VALUE) {
 +          tabletTime.useMaxTimeFromWALog(count[1]);
 +        }
 +        commitSession.updateMaxCommittedTime(tabletTime.getTime());
 +
 +        if (count[0] == 0) {
 +          log.debug("No replayed mutations applied, removing unused entries for " + extent);
 +          MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
 +          logEntries.clear();
 +        }
 +
 +      } catch (Throwable t) {
 +        if (acuTableConf.getBoolean(Property.TABLE_FAILURES_IGNORE)) {
 +          log.warn("Error recovering from log files: ", t);
 +        } else {
 +          throw new RuntimeException(t);
 +        }
 +      }
 +      // make some closed references that represent the recovered logs
 +      currentLogs = new HashSet<DfsLogger>();
 +      for (LogEntry logEntry : logEntries) {
 +        for (String log : logEntry.logSet) {
 +          currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), log, logEntry.getColumnQualifier().toString()));
 +        }
 +      }
 +
 +      log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries()
 +          + " entries created)");
 +    }
 +
 +    String contextName = acuTableConf.get(Property.TABLE_CLASSPATH);
 +    if (contextName != null && !contextName.equals("")) {
 +      // initialize context classloader, instead of possibly waiting for it to initialize for a scan
 +      // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292
 +      AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
 +    }
 +
 +    // do this last after tablet is completely setup because it
 +    // could cause major compaction to start
 +    datafileManager = new DatafileManager(datafiles);
 +
 +    computeNumEntries();
 +
 +    datafileManager.removeFilesAfterScan(scanFiles);
 +
 +    // look for hints of a failure on the previous tablet server
 +    if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
 +      // look for any temp files hanging around
 +      removeOldTemporaryFiles();
 +    }
 +
 +    log.log(TLevel.TABLET_HIST, extent + " opened");
 +  }
 +
 +  private void removeOldTemporaryFiles() {
 +    // remove any temporary files created by a previous tablet server
 +    try {
 +      for (FileStatus tmp : fs.globStatus(new Path(location, "*_tmp"))) {
 +        try {
 +          log.debug("Removing old temp file " + tmp.getPath());
 +          fs.delete(tmp.getPath());
 +        } catch (IOException ex) {
 +          log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
 +        }
 +      }
 +    } catch (IOException ex) {
 +      log.error("Error scanning for old temp files in " + location);
 +    }
 +  }
 +
 +  private void setupDefaultSecurityLabels(KeyExtent extent) {
 +    if (extent.isMeta()) {
 +      defaultSecurityLabel = new byte[0];
 +    } else {
 +      try {
 +        ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
 +        this.defaultSecurityLabel = cv.getExpression();
 +      } catch (Exception e) {
 +        log.error(e, e);
 +        this.defaultSecurityLabel = new byte[0];
 +      }
 +    }
 +  }
 +
 +  public static class KVEntry extends KeyValue {
 +    private static final long serialVersionUID = 1L;
 +
 +    public KVEntry(Key k, Value v) {
 +      super(new Key(k), Arrays.copyOf(v.get(), v.get().length));
 +    }
 +
 +    int numBytes() {
 +      return getKey().getSize() + getValue().get().length;
 +    }
 +
 +    int estimateMemoryUsed() {
 +      return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object
 +    }
 +  }
 +
 +  private LookupResult lookup(SortedKeyValueIterator<Key,Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, ArrayList<KVEntry> results,
 +      long maxResultsSize) throws IOException {
 +
 +    LookupResult lookupResult = new LookupResult();
 +
 +    boolean exceededMemoryUsage = false;
 +    boolean tabletClosed = false;
 +
 +    Set<ByteSequence> cfset = null;
 +    if (columnSet.size() > 0)
 +      cfset = LocalityGroupUtil.families(columnSet);
 +
 +    for (Range range : ranges) {
 +
 +      if (exceededMemoryUsage || tabletClosed) {
 +        lookupResult.unfinishedRanges.add(range);
 +        continue;
 +      }
 +
 +      int entriesAdded = 0;
 +
 +      try {
 +        if (cfset != null)
 +          mmfi.seek(range, cfset, true);
 +        else
 +          mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
 +
 +        while (mmfi.hasTop()) {
 +          Key key = mmfi.getTopKey();
 +
 +          KVEntry kve = new KVEntry(key, mmfi.getTopValue());
 +          results.add(kve);
 +          entriesAdded++;
 +          lookupResult.bytesAdded += kve.estimateMemoryUsed();
 +          lookupResult.dataSize += kve.numBytes();
 +
 +          exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
 +
 +          if (exceededMemoryUsage) {
 +            addUnfinishedRange(lookupResult, range, key, false);
 +            break;
 +          }
 +
 +          mmfi.next();
 +        }
 +
 +      } catch (TooManyFilesException tmfe) {
 +        // treat this as a closed tablet, and let the client retry
 +        log.warn("Tablet " + getExtent() + " has too many files, batch lookup can not run");
 +        handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +        tabletClosed = true;
 +      } catch (IOException ioe) {
 +        if (shutdownInProgress()) {
 +          // assume HDFS shutdown hook caused this exception
 +          log.debug("IOException while shutdown in progress ", ioe);
 +          handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +          tabletClosed = true;
 +        } else {
 +          throw ioe;
 +        }
 +      } catch (IterationInterruptedException iie) {
 +        if (isClosed()) {
 +          handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +          tabletClosed = true;
 +        } else {
 +          throw iie;
 +        }
 +      } catch (TabletClosedException tce) {
 +        handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +        tabletClosed = true;
 +      }
 +
 +    }
 +
 +    return lookupResult;
 +  }
 +
 +  private void handleTabletClosedDuringScan(ArrayList<KVEntry> results, LookupResult lookupResult, boolean exceededMemoryUsage, Range range, int entriesAdded) {
 +    if (exceededMemoryUsage)
 +      throw new IllegalStateException("tablet should not exceed memory usage or close, not both");
 +
 +    if (entriesAdded > 0)
 +      addUnfinishedRange(lookupResult, range, results.get(results.size() - 1).getKey(), false);
 +    else
 +      lookupResult.unfinishedRanges.add(range);
 +
 +    lookupResult.closed = true;
 +  }
 +
 +  private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) {
 +    if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) {
 +      Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive());
 +      lookupResult.unfinishedRanges.add(nlur);
 +    }
 +  }
 +
 +  public static interface KVReceiver {
 +    void receive(List<KVEntry> matches) throws IOException;
 +  }
 +
 +  class LookupResult {
 +    List<Range> unfinishedRanges = new ArrayList<Range>();
 +    long bytesAdded = 0;
 +    long dataSize = 0;
 +    boolean closed = false;
 +  }
 +
 +  public LookupResult lookup(List<Range> ranges, HashSet<Column> columns, Authorizations authorizations, ArrayList<KVEntry> results, long maxResultSize,
 +      List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag) throws IOException {
 +
 +    if (ranges.size() == 0) {
 +      return new LookupResult();
 +    }
 +
 +    ranges = Range.mergeOverlapping(ranges);
 +    Collections.sort(ranges);
 +
 +    Range tabletRange = extent.toDataRange();
 +    for (Range range : ranges) {
 +      // do a test to see if this range falls within the tablet, if it does not
 +      // then clip will throw an exception
 +      tabletRange.clip(range);
 +    }
 +
 +    ScanDataSource dataSource = new ScanDataSource(authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
 +
 +    LookupResult result = null;
 +
 +    try {
 +      SortedKeyValueIterator<Key,Value> iter = new SourceSwitchingIterator(dataSource);
 +      result = lookup(iter, ranges, columns, results, maxResultSize);
 +      return result;
 +    } catch (IOException ioe) {
 +      dataSource.close(true);
 +      throw ioe;
 +    } finally {
 +      // code in finally block because always want
 +      // to return mapfiles, even when exception is thrown
 +      dataSource.close(false);
 +
 +      synchronized (this) {
 +        queryCount += results.size();
 +        if (result != null)
 +          queryBytes += result.dataSize;
 +      }
 +    }
 +  }
 +
 +  private Batch nextBatch(SortedKeyValueIterator<Key,Value> iter, Range range, int num, Set<Column> columns) throws IOException {
 +
 +    // log.info("In nextBatch..");
 +
 +    List<KVEntry> results = new ArrayList<KVEntry>();
 +    Key key = null;
 +
 +    Value value;
 +    long resultSize = 0L;
 +    long resultBytes = 0L;
 +
 +    long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
 +
 +    if (columns.size() == 0) {
 +      iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
 +    } else {
 +      iter.seek(range, LocalityGroupUtil.families(columns), true);
 +    }
 +
 +    Key continueKey = null;
 +    boolean skipContinueKey = false;
 +
 +    boolean endOfTabletReached = false;
 +    while (iter.hasTop()) {
 +
 +      value = iter.getTopValue();
 +      key = iter.getTopKey();
 +
 +      KVEntry kvEntry = new KVEntry(key, value); // copies key and value
 +      results.add(kvEntry);
 +      resultSize += kvEntry.estimateMemoryUsed();
 +      resultBytes += kvEntry.numBytes();
 +
 +      if (resultSize >= maxResultsSize || results.size() >= num) {
 +        continueKey = new Key(key);
 +        skipContinueKey = true;
 +        break;
 +      }
 +
 +      iter.next();
 +    }
 +
 +    if (iter.hasTop() == false) {
 +      endOfTabletReached = true;
 +    }
 +
 +    Batch retBatch = new Batch();
 +    retBatch.numBytes = resultBytes;
 +
 +    if (!endOfTabletReached) {
 +      retBatch.continueKey = continueKey;
 +      retBatch.skipContinueKey = skipContinueKey;
 +    } else {
 +      retBatch.continueKey = null;
 +    }
 +
 +    if (endOfTabletReached && results.size() == 0)
 +      retBatch.results = null;
 +    else
 +      retBatch.results = results;
 +
 +    return retBatch;
 +  }
 +
 +  /**
 +   * Determine if a JVM shutdown is in progress.
 +   *
 +   */
 +  private boolean shutdownInProgress() {
 +    try {
 +      Runtime.getRuntime().removeShutdownHook(new Thread(new Runnable() {
 +        @Override
 +        public void run() {}
 +      }));
 +    } catch (IllegalStateException ise) {
 +      return true;
 +    }
 +
 +    return false;
 +  }
 +
 +  private class Batch {
 +    public boolean skipContinueKey;
 +    public List<KVEntry> results;
 +    public Key continueKey;
 +    public long numBytes;
 +  }
 +
 +  Scanner createScanner(Range range, int num, Set<Column> columns, Authorizations authorizations, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
 +      boolean isolated, AtomicBoolean interruptFlag) {
 +    // do a test to see if this range falls within the tablet, if it does not
 +    // then clip will throw an exception
 +    extent.toDataRange().clip(range);
 +
 +    ScanOptions opts = new ScanOptions(num, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag, isolated);
 +    return new Scanner(range, opts);
 +  }
 +
 +  class ScanBatch {
 +    boolean more;
 +    List<KVEntry> results;
 +
 +    ScanBatch(List<KVEntry> results, boolean more) {
 +      this.results = results;
 +      this.more = more;
 +    }
 +  }
 +
 +  class Scanner {
 +
 +    private ScanOptions options;
 +    private Range range;
 +    private SortedKeyValueIterator<Key,Value> isolatedIter;
 +    private ScanDataSource isolatedDataSource;
 +    private boolean sawException = false;
 +    private boolean scanClosed = false;
 +
 +    Scanner(Range range, ScanOptions options) {
 +      this.range = range;
 +      this.options = options;
 +    }
 +
 +    synchronized ScanBatch read() throws IOException, TabletClosedException {
 +
 +      if (sawException)
 +        throw new IllegalStateException("Tried to use scanner after exception occurred.");
 +
 +      if (scanClosed)
 +        throw new IllegalStateException("Tried to use scanner after it was closed.");
 +
 +      Batch results = null;
 +
 +      ScanDataSource dataSource;
 +
 +      if (options.isolated) {
 +        if (isolatedDataSource == null)
 +          isolatedDataSource = new ScanDataSource(options);
 +        dataSource = isolatedDataSource;
 +      } else {
 +        dataSource = new ScanDataSource(options);
 +      }
 +
 +      try {
 +
 +        SortedKeyValueIterator<Key,Value> iter;
 +
 +        if (options.isolated) {
 +          if (isolatedIter == null)
 +            isolatedIter = new SourceSwitchingIterator(dataSource, true);
 +          else
 +            isolatedDataSource.fileManager.reattach();
 +          iter = isolatedIter;
 +        } else {
 +          iter = new SourceSwitchingIterator(dataSource, false);
 +        }
 +
 +        results = nextBatch(iter, range, options.num, options.columnSet);
 +
 +        if (results.results == null) {
 +          range = null;
 +          return new ScanBatch(new ArrayList<Tablet.KVEntry>(), false);
 +        } else if (results.continueKey == null) {
 +          return new ScanBatch(results.results, false);
 +        } else {
 +          range = new Range(results.continueKey, !results.skipContinueKey, range.getEndKey(), range.isEndKeyInclusive());
 +          return new ScanBatch(results.results, true);
 +        }
 +
 +      } catch (IterationInterruptedException iie) {
 +        sawException = true;
 +        if (isClosed())
 +          throw new TabletClosedException(iie);
 +        else
 +          throw iie;
 +      } catch (IOException ioe) {
 +        if (shutdownInProgress()) {
 +          log.debug("IOException while shutdown in progress ", ioe);
 +          throw new TabletClosedException(ioe); // assume IOException was caused by execution of HDFS shutdown hook
 +        }
 +
 +        sawException = true;
 +        dataSource.close(true);
 +        throw ioe;
 +      } catch (RuntimeException re) {
 +        sawException = true;
 +        throw re;
 +      } finally {
 +        // code in finally block because always want
 +        // to return mapfiles, even when exception is thrown
 +        if (!options.isolated)
 +          dataSource.close(false);
 +        else if (dataSource.fileManager != null)
 +          dataSource.fileManager.detach();
 +
 +        synchronized (Tablet.this) {
 +          if (results != null && results.results != null) {
 +            long more = results.results.size();
 +            queryCount += more;
 +            queryBytes += results.numBytes;
 +          }
 +        }
 +      }
 +    }
 +
 +    // close and read are synchronized because can not call close on the data source while it is in use
 +    // this cloud lead to the case where file iterators that are in use by a thread are returned
 +    // to the pool... this would be bad
 +    void close() {
 +      options.interruptFlag.set(true);
 +      synchronized (this) {
 +        scanClosed = true;
 +        if (isolatedDataSource != null)
 +          isolatedDataSource.close(false);
 +      }
 +    }
 +  }
 +
 +  static class ScanOptions {
 +
 +    // scan options
 +    Authorizations authorizations;
 +    byte[] defaultLabels;
 +    Set<Column> columnSet;
 +    List<IterInfo> ssiList;
 +    Map<String,Map<String,String>> ssio;
 +    AtomicBoolean interruptFlag;
 +    int num;
 +    boolean isolated;
 +
 +    ScanOptions(int num, Authorizations authorizations, byte[] defaultLabels, Set<Column> columnSet, List<IterInfo> ssiList,
 +        Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag, boolean isolated) {
 +      this.num = num;
 +      this.authorizations = authorizations;
 +      this.defaultLabels = defaultLabels;
 +      this.columnSet = columnSet;
 +      this.ssiList = ssiList;
 +      this.ssio = ssio;
 +      this.interruptFlag = interruptFlag;
 +      this.isolated = isolated;
 +    }
 +
 +  }
 +
 +  class ScanDataSource implements DataSource {
 +
 +    // data source state
 +    private ScanFileManager fileManager;
 +    private SortedKeyValueIterator<Key,Value> iter;
 +    private long expectedDeletionCount;
 +    private List<MemoryIterator> memIters = null;
 +    private long fileReservationId;
 +    private AtomicBoolean interruptFlag;
 +    private StatsIterator statsIterator;
 +
 +    ScanOptions options;
 +
 +    ScanDataSource(Authorizations authorizations, byte[] defaultLabels, HashSet<Column> columnSet, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
 +        AtomicBoolean interruptFlag) {
 +      expectedDeletionCount = dataSourceDeletions.get();
 +      this.options = new ScanOptions(-1, authorizations, defaultLabels, columnSet, ssiList, ssio, interruptFlag, false);
 +      this.interruptFlag = interruptFlag;
 +    }
 +
 +    ScanDataSource(ScanOptions options) {
 +      expectedDeletionCount = dataSourceDeletions.get();
 +      this.options = options;
 +      this.interruptFlag = options.interruptFlag;
 +    }
 +
 +    @Override
 +    public DataSource getNewDataSource() {
 +      if (!isCurrent()) {
 +        // log.debug("Switching data sources during a scan");
 +        if (memIters != null) {
 +          tabletMemory.returnIterators(memIters);
 +          memIters = null;
 +          datafileManager.returnFilesForScan(fileReservationId);
 +          fileReservationId = -1;
 +        }
 +
 +        if (fileManager != null)
 +          fileManager.releaseOpenFiles(false);
 +
 +        expectedDeletionCount = dataSourceDeletions.get();
 +        iter = null;
 +
 +        return this;
 +      } else
 +        return this;
 +    }
 +
 +    @Override
 +    public boolean isCurrent() {
 +      return expectedDeletionCount == dataSourceDeletions.get();
 +    }
 +
 +    @Override
 +    public SortedKeyValueIterator<Key,Value> iterator() throws IOException {
 +      if (iter == null)
 +        iter = createIterator();
 +      return iter;
 +    }
 +
 +    private SortedKeyValueIterator<Key,Value> createIterator() throws IOException {
 +
 +      Map<FileRef,DataFileValue> files;
 +
 +      synchronized (Tablet.this) {
 +
 +        if (memIters != null)
 +          throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
 +
 +        if (Tablet.this.closed)
 +          throw new TabletClosedException();
 +
 +        if (interruptFlag.get())
 +          throw new IterationInterruptedException(extent.toString() + " " + interruptFlag.hashCode());
 +
 +        // only acquire the file manager when we know the tablet is open
 +        if (fileManager == null) {
 +          fileManager = tabletResources.newScanFileManager();
 +          activeScans.add(this);
 +        }
 +
 +        if (fileManager.getNumOpenFiles() != 0)
 +          throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
 +
 +        // set this before trying to get iterators in case
 +        // getIterators() throws an exception
 +        expectedDeletionCount = dataSourceDeletions.get();
 +
 +        memIters = tabletMemory.getIterators();
 +        Pair<Long,Map<FileRef,DataFileValue>> reservation = datafileManager.reserveFilesForScan();
 +        fileReservationId = reservation.getFirst();
 +        files = reservation.getSecond();
 +      }
 +
 +      Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, options.isolated);
 +
 +      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.size() + memIters.size());
 +
 +      iters.addAll(mapfiles);
 +      iters.addAll(memIters);
 +
 +      for (SortedKeyValueIterator<Key,Value> skvi : iters)
 +        ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
 +
 +      MultiIterator multiIter = new MultiIterator(iters, extent);
 +
 +      TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager, files);
 +
 +      statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, scannedCount);
 +
 +      DeletingIterator delIter = new DeletingIterator(statsIterator, false);
 +
 +      ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
 +
 +      ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, options.columnSet);
 +
 +      VisibilityFilter visFilter = new VisibilityFilter(colFilter, options.authorizations, options.defaultLabels);
 +
 +      return iterEnv.getTopLevelIterator(IteratorUtil
 +          .loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.ssiList, options.ssio, iterEnv));
 +    }
 +
 +    private void close(boolean sawErrors) {
 +
 +      if (memIters != null) {
 +        tabletMemory.returnIterators(memIters);
 +        memIters = null;
 +        datafileManager.returnFilesForScan(fileReservationId);
 +        fileReservationId = -1;
 +      }
 +
 +      synchronized (Tablet.this) {
 +        activeScans.remove(this);
 +        if (activeScans.size() == 0)
 +          Tablet.this.notifyAll();
 +      }
 +
 +      if (fileManager != null) {
 +        fileManager.releaseOpenFiles(sawErrors);
 +        fileManager = null;
 +      }
 +
 +      if (statsIterator != null) {
 +        statsIterator.report();
 +      }
 +
 +    }
 +
 +    public void interrupt() {
 +      interruptFlag.set(true);
 +    }
 +
 +    @Override
 +    public DataSource getDeepCopyDataSource(IteratorEnvironment env) {
 +      throw new UnsupportedOperationException();
 +    }
 +
 +    @Override
 +    public void setInterruptFlag(AtomicBoolean flag) {
 +      throw new UnsupportedOperationException();
 +    }
 +
 +  }
 +
 +  private DataFileValue minorCompact(Configuration conf, VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile,
 +      boolean hasQueueTime, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
 +    boolean failed = false;
 +    long start = System.currentTimeMillis();
 +    timer.incrementStatusMinor();
 +
 +    long count = 0;
 +
 +    try {
 +      Span span = Trace.start("write");
 +      CompactionStats stats;
 +      try {
 +        count = memTable.getNumEntries();
 +
 +        DataFileValue dfv = null;
 +        if (mergeFile != null)
 +          dfv = datafileManager.getDatafileSizes().get(mergeFile);
 +
 +        MinorCompactor compactor = new MinorCompactor(conf, fs, memTable, mergeFile, dfv, tmpDatafile, acuTableConf, extent, mincReason);
 +        stats = compactor.call();
 +      } finally {
 +        span.stop();
 +      }
 +      span = Trace.start("bringOnline");
 +      try {
 +        datafileManager.bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
 +            commitSession, flushId);
 +      } finally {
 +        span.stop();
 +      }
 +      return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
 +    } catch (Exception E) {
 +      failed = true;
 +      throw new RuntimeException(E);
 +    } catch (Error E) {
 +      // Weird errors like "OutOfMemoryError" when trying to create the thread for the compaction
 +      failed = true;
 +      throw new RuntimeException(E);
 +    } finally {
 +      try {
 +        tabletMemory.finalizeMinC();
 +      } catch (Throwable t) {
 +        log.error("Failed to free tablet memory", t);
 +      }
 +
 +      if (!failed) {
 +        lastMinorCompactionFinishTime = System.currentTimeMillis();
 +      }
 +      if (tabletServer.mincMetrics.isEnabled())
 +        tabletServer.mincMetrics.add(TabletServerMinCMetrics.minc, (lastMinorCompactionFinishTime - start));
 +      if (hasQueueTime) {
 +        timer.updateTime(Operation.MINOR, queued, start, count, failed);
 +        if (tabletServer.mincMetrics.isEnabled())
 +          tabletServer.mincMetrics.add(TabletServerMinCMetrics.queue, (start - queued));
 +      } else
 +        timer.updateTime(Operation.MINOR, start, count, failed);
 +    }
 +  }
 +
 +  private class MinorCompactionTask implements Runnable {
 +
 +    private long queued;
 +    private CommitSession commitSession;
 +    private DataFileValue stats;
 +    private FileRef mergeFile;
 +    private long flushId;
 +    private MinorCompactionReason mincReason;
 +
 +    MinorCompactionTask(FileRef mergeFile, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
 +      queued = System.currentTimeMillis();
 +      minorCompactionWaitingToStart = true;
 +      this.commitSession = commitSession;
 +      this.mergeFile = mergeFile;
 +      this.flushId = flushId;
 +      this.mincReason = mincReason;
 +    }
 +
 +    @Override
 +    public void run() {
 +      minorCompactionWaitingToStart = false;
 +      minorCompactionInProgress = true;
 +      Span minorCompaction = Trace.on("minorCompaction");
 +      try {
 +        FileRef newMapfileLocation = getNextMapFilename(mergeFile == null ? "F" : "M");
 +        FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp");
 +        Span span = Trace.start("waitForCommits");
 +        synchronized (Tablet.this) {
 +          commitSession.waitForCommitsToFinish();
 +        }
 +        span.stop();
 +        span = Trace.start("start");
 +        while (true) {
 +          try {
 +            // the purpose of the minor compaction start event is to keep track of the filename... in the case
 +            // where the metadata table write for the minor compaction finishes and the process dies before
 +            // writing the minor compaction finish event, then the start event+filename in metadata table will
 +            // prevent recovery of duplicate data... the minor compaction start event could be written at any time
 +            // before the metadata write for the minor compaction
 +            tabletServer.minorCompactionStarted(commitSession, commitSession.getWALogSeq() + 1, newMapfileLocation.path().toString());
 +            break;
 +          } catch (IOException e) {
 +            log.warn("Failed to write to write ahead log " + e.getMessage(), e);
 +          }
 +        }
 +        span.stop();
 +        span = Trace.start("compact");
 +        this.stats = minorCompact(conf, fs, tabletMemory.getMinCMemTable(), tmpFileRef, newMapfileLocation, mergeFile, true, queued, commitSession, flushId,
 +            mincReason);
 +        span.stop();
 +
 +        if (needsSplit()) {
 +          tabletServer.executeSplit(Tablet.this);
 +        } else {
 +          initiateMajorCompaction(MajorCompactionReason.NORMAL);
 +        }
 +      } catch (Throwable t) {
 +        log.error("Unknown error during minor compaction for extent: " + getExtent(), t);
 +        throw new RuntimeException(t);
 +      } finally {
 +        minorCompactionInProgress = false;
 +        minorCompaction.data("extent", extent.toString());
 +        minorCompaction.data("numEntries", Long.toString(this.stats.getNumEntries()));
 +        minorCompaction.data("size", Long.toString(this.stats.getSize()));
 +        minorCompaction.stop();
 +      }
 +    }
 +  }
 +
 +  private synchronized MinorCompactionTask prepareForMinC(long flushId, MinorCompactionReason mincReason) {
 +    CommitSession oldCommitSession = tabletMemory.prepareForMinC();
 +    otherLogs = currentLogs;
 +    currentLogs = new HashSet<DfsLogger>();
 +
 +    FileRef mergeFile = null;
 +    if (mincReason != MinorCompactionReason.RECOVERY) {
 +      mergeFile = datafileManager.reserveMergingMinorCompactionFile();
 +    }
 +
 +    return new MinorCompactionTask(mergeFile, oldCommitSession, flushId, mincReason);
 +
 +  }
 +
 +  void flush(long tableFlushID) {
 +    boolean updateMetadata = false;
 +    boolean initiateMinor = false;
 +
 +    try {
 +
 +      synchronized (this) {
 +
 +        // only want one thing at a time to update flush ID to ensure that metadata table and tablet in memory state are consistent
 +        if (updatingFlushID)
 +          return;
 +
 +        if (lastFlushID >= tableFlushID)
 +          return;
 +
 +        if (closing || closed || tabletMemory.memoryReservedForMinC())
 +          return;
 +
 +        if (tabletMemory.getMemTable().getNumEntries() == 0) {
 +          lastFlushID = tableFlushID;
 +          updatingFlushID = true;
 +          updateMetadata = true;
 +        } else
 +          initiateMinor = true;
 +      }
 +
 +      if (updateMetadata) {
 +        Credentials creds = SystemCredentials.get();
 +        // if multiple threads were allowed to update this outside of a sync block, then it would be
 +        // a race condition
 +        MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, tabletServer.getLock());
 +      } else if (initiateMinor)
 +        initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
 +
 +    } finally {
 +      if (updateMetadata) {
 +        synchronized (this) {
 +          updatingFlushID = false;
 +          this.notifyAll();
 +        }
 +      }
 +    }
 +
 +  }
 +
 +  boolean initiateMinorCompaction(MinorCompactionReason mincReason) {
 +    if (isClosed()) {
 +      // don't bother trying to get flush id if closed... could be closed after this check but that is ok... just trying to cut down on uneeded log messages....
 +      return false;
 +    }
 +
 +    // get the flush id before the new memmap is made available for write
 +    long flushId;
 +    try {
 +      flushId = getFlushID();
 +    } catch (NoNodeException e) {
 +      log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
 +      return false;
 +    }
 +    return initiateMinorCompaction(flushId, mincReason);
 +  }
 +
 +  boolean minorCompactNow(MinorCompactionReason mincReason) {
 +    long flushId;
 +    try {
 +      flushId = getFlushID();
 +    } catch (NoNodeException e) {
 +      log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
 +      return false;
 +    }
 +    MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
 +    if (mct == null)
 +      return false;
 +    mct.run();
 +    return true;
 +  }
 +
 +  boolean initiateMinorCompaction(long flushId, MinorCompactionReason mincReason) {
 +    MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
 +    if (mct == null)
 +      return false;
 +    tabletResources.executeMinorCompaction(mct);
 +    return true;
 +  }
 +
 +  private MinorCompactionTask createMinorCompactionTask(long flushId, MinorCompactionReason mincReason) {
 +    MinorCompactionTask mct;
 +    long t1, t2;
 +
 +    StringBuilder logMessage = null;
 +
 +    try {
 +      synchronized (this) {
 +        t1 = System.currentTimeMillis();
 +
 +        if (closing || closed || majorCompactionWaitingToStart || tabletMemory.memoryReservedForMinC() || tabletMemory.getMemTable().getNumEntries() == 0
 +            || updatingFlushID) {
 +
 +          logMessage = new StringBuilder();
 +
 +          logMessage.append(extent.toString());
 +          logMessage.append(" closing " + closing);
 +          logMessage.append(" closed " + closed);
 +          logMessage.append(" majorCompactionWaitingToStart " + majorCompactionWaitingToStart);
 +          if (tabletMemory != null)
 +            logMessage.append(" tabletMemory.memoryReservedForMinC() " + tabletMemory.memoryReservedForMinC());
 +          if (tabletMemory != null && tabletMemory.getMemTable() != null)
 +            logMessage.append(" tabletMemory.getMemTable().getNumEntries() " + tabletMemory.getMemTable().getNumEntries());
 +          logMessage.append(" updatingFlushID " + updatingFlushID);
 +
 +          return null;
 +        }
 +
 +        mct = prepareForMinC(flushId, mincReason);
 +        t2 = System.currentTimeMillis();
 +      }
 +    } finally {
 +      // log outside of sync block
 +      if (logMessage != null && log.isDebugEnabled())
 +        log.debug(logMessage);
 +    }
 +
 +    log.debug(String.format("MinC initiate lock %.2f secs", (t2 - t1) / 1000.0));
 +    return mct;
 +  }
 +
 +  long getFlushID() throws NoNodeException {
 +    try {
 +      String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +          + Constants.ZTABLE_FLUSH_ID;
-       return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++      return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    } catch (NumberFormatException nfe) {
 +      throw new RuntimeException(nfe);
 +    } catch (KeeperException ke) {
 +      if (ke instanceof NoNodeException) {
 +        throw (NoNodeException) ke;
 +      } else {
 +        throw new RuntimeException(ke);
 +      }
 +    }
 +  }
 +
 +  long getCompactionCancelID() {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    try {
-       return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++      return Long.parseLong(new String(ZooReaderWriter.getInstance().get

<TRUNCATED>

[22/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
index c07ed1c,0000000..d77d060
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
@@@ -1,193 -1,0 +1,194 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.MemoryUnit;
 +import org.apache.accumulo.minicluster.ServerType;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.test.TestIngest;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.junit.Assert;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class LargeRowIT extends ConfigurableMacIT {
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 2, MemoryUnit.BYTE);
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "10ms"));
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 4 * 60;
 +  }
 +
 +  private static final int SEED = 42;
 +  private static final String REG_TABLE_NAME = "lr";
 +  private static final String PRE_SPLIT_TABLE_NAME = "lrps";
 +  private static final int NUM_ROWS = 100;
 +  private static final int ROW_SIZE = 1 << 17;
 +  private static final int NUM_PRE_SPLITS = 9;
 +  private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / NUM_PRE_SPLITS;
 +
 +  private int timeoutFactor = 1;
 +
 +  @Before
 +  public void getTimeoutFactor() {
 +    try {
 +      timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
 +    } catch (NumberFormatException e) {
 +      log.warn("Could not parse property value for 'timeout.factor' as integer: " + System.getProperty("timeout.factor"));
 +    }
 +
 +    Assert.assertTrue("Timeout factor must be greater than or equal to 1", timeoutFactor >= 1);
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Random r = new Random();
 +    byte rowData[] = new byte[ROW_SIZE];
 +    r.setSeed(SEED + 1);
 +    TreeSet<Text> splitPoints = new TreeSet<Text>();
 +    for (int i = 0; i < NUM_PRE_SPLITS; i++) {
 +      r.nextBytes(rowData);
 +      TestIngest.toPrintableChars(rowData);
 +      splitPoints.add(new Text(rowData));
 +    }
 +    Connector c = getConnector();
 +    c.tableOperations().create(REG_TABLE_NAME);
 +    c.tableOperations().create(PRE_SPLIT_TABLE_NAME);
 +    c.tableOperations().addSplits(PRE_SPLIT_TABLE_NAME, splitPoints);
 +    test1(c);
 +    test2(c);
 +  }
 +
 +  private void test1(Connector c) throws Exception {
 +
 +    basicTest(c, REG_TABLE_NAME, 0);
 +
 +    c.tableOperations().setProperty(REG_TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "" + SPLIT_THRESH);
 +
 +    UtilWaitThread.sleep(timeoutFactor * 12000);
 +    Logger.getLogger(LargeRowIT.class).warn("checking splits");
 +    FunctionalTestUtils.checkSplits(c, REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
 +
 +    verify(c, REG_TABLE_NAME);
 +  }
 +
 +  private void test2(Connector c) throws Exception {
 +    basicTest(c, PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
 +  }
 +
 +  private void basicTest(Connector c, String table, int expectedSplits) throws Exception {
 +    BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
 +
 +    Random r = new Random();
 +    byte rowData[] = new byte[ROW_SIZE];
 +
 +    r.setSeed(SEED);
 +
 +    for (int i = 0; i < NUM_ROWS; i++) {
 +
 +      r.nextBytes(rowData);
 +      TestIngest.toPrintableChars(rowData);
 +
 +      Mutation mut = new Mutation(new Text(rowData));
-       mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(UTF_8)));
 +      bw.addMutation(mut);
 +    }
 +
 +    bw.close();
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +
 +    verify(c, table);
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +
 +    c.tableOperations().flush(table, null, null, false);
 +
 +    // verify while table flush is running
 +    verify(c, table);
 +
 +    // give split time to complete
 +    c.tableOperations().flush(table, null, null, true);
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +
 +    verify(c, table);
 +
 +    FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
 +  }
 +
 +  private void verify(Connector c, String table) throws Exception {
 +    Random r = new Random();
 +    byte rowData[] = new byte[ROW_SIZE];
 +
 +    r.setSeed(SEED);
 +
 +    Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
 +
 +    for (int i = 0; i < NUM_ROWS; i++) {
 +
 +      r.nextBytes(rowData);
 +      TestIngest.toPrintableChars(rowData);
 +
 +      scanner.setRange(new Range(new Text(rowData)));
 +
 +      int count = 0;
 +
 +      for (Entry<Key,Value> entry : scanner) {
 +        if (!entry.getKey().getRow().equals(new Text(rowData))) {
 +          throw new Exception("verification failed, unexpected row i =" + i);
 +        }
-         if (!entry.getValue().equals(Integer.toString(i).getBytes(Constants.UTF8))) {
++        if (!entry.getValue().equals(Integer.toString(i).getBytes(UTF_8))) {
 +          throw new Exception("verification failed, unexpected value i =" + i + " value = " + entry.getValue());
 +        }
 +        count++;
 +      }
 +
 +      if (count != 1) {
 +        throw new Exception("verification failed, unexpected count i =" + i + " count=" + count);
 +      }
 +
 +    }
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 3102f59,0000000..f4bee97
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -1,612 -1,0 +1,613 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.File;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.NoSuchElementException;
 +import java.util.Random;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.tserver.NativeMap;
 +import org.apache.hadoop.io.Text;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +public class NativeMapIT {
 +
 +  private Key nk(int r) {
 +    return new Key(new Text(String.format("r%09d", r)));
 +  }
 +
 +  private Key nk(int r, int cf, int cq, int cv, int ts, boolean deleted) {
 +    Key k = new Key(new Text(String.format("r%09d", r)), new Text(String.format("cf%09d", cf)), new Text(String.format("cq%09d", cq)), new Text(String.format(
 +        "cv%09d", cv)), ts);
 +
 +    k.setDeleted(deleted);
 +
 +    return k;
 +  }
 +
 +  private Value nv(int v) {
-     return new Value(String.format("r%09d", v).getBytes(Constants.UTF8));
++    return new Value(String.format("r%09d", v).getBytes(UTF_8));
 +  }
 +
 +  public static File nativeMapLocation() {
 +    File projectDir = new File(System.getProperty("user.dir")).getParentFile();
 +    File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION);
 +    return nativeMapDir;
 +  }
 +
 +  @BeforeClass
 +  public static void setUp() {
 +    NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
 +  }
 +
 +  private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {
 +    for (int i = start; i <= end; i++) {
 +      assertTrue(iter.hasNext());
 +      Entry<Key,Value> entry = iter.next();
 +      assertEquals(nk(i), entry.getKey());
 +      assertEquals(nv(i + valueOffset), entry.getValue());
 +    }
 +
 +    assertFalse(iter.hasNext());
 +  }
 +
 +  private void insertAndVerify(NativeMap nm, int start, int end, int valueOffset) {
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i + valueOffset));
 +    }
 +
 +    for (int i = start; i <= end; i++) {
 +      Value v = nm.get(nk(i));
 +      assertNotNull(v);
 +      assertEquals(nv(i + valueOffset), v);
 +
 +      Iterator<Entry<Key,Value>> iter2 = nm.iterator(nk(i));
 +      assertTrue(iter2.hasNext());
 +      Entry<Key,Value> entry = iter2.next();
 +      assertEquals(nk(i), entry.getKey());
 +      assertEquals(nv(i + valueOffset), entry.getValue());
 +    }
 +
 +    assertNull(nm.get(nk(start - 1)));
 +
 +    assertNull(nm.get(nk(end + 1)));
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    verifyIterator(start, end, valueOffset, iter);
 +
 +    for (int i = start; i <= end; i++) {
 +      iter = nm.iterator(nk(i));
 +      verifyIterator(i, end, valueOffset, iter);
 +
 +      // lookup nonexistant key that falls after existing key
 +      iter = nm.iterator(nk(i, 1, 1, 1, 1, false));
 +      verifyIterator(i + 1, end, valueOffset, iter);
 +    }
 +
 +    assertEquals(end - start + 1, nm.size());
 +  }
 +
 +  private void insertAndVerifyExhaustive(NativeMap nm, int num, int run) {
 +    for (int i = 0; i < num; i++) {
 +      for (int j = 0; j < num; j++) {
 +        for (int k = 0; k < num; k++) {
 +          for (int l = 0; l < num; l++) {
 +            for (int ts = 0; ts < num; ts++) {
 +              Key key = nk(i, j, k, l, ts, true);
-               Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++              Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
 +
 +              nm.put(key, value);
 +
 +              key = nk(i, j, k, l, ts, false);
-               value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++              value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
 +
 +              nm.put(key, value);
 +            }
 +          }
 +        }
 +      }
 +    }
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +
 +    for (int i = 0; i < num; i++) {
 +      for (int j = 0; j < num; j++) {
 +        for (int k = 0; k < num; k++) {
 +          for (int l = 0; l < num; l++) {
 +            for (int ts = num - 1; ts >= 0; ts--) {
 +              Key key = nk(i, j, k, l, ts, true);
-               Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++              Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
 +
 +              assertTrue(iter.hasNext());
 +              Entry<Key,Value> entry = iter.next();
 +              assertEquals(key, entry.getKey());
 +              assertEquals(value, entry.getValue());
 +
 +              key = nk(i, j, k, l, ts, false);
-               value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++              value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
 +
 +              assertTrue(iter.hasNext());
 +              entry = iter.next();
 +              assertEquals(key, entry.getKey());
 +              assertEquals(value, entry.getValue());
 +            }
 +          }
 +        }
 +      }
 +    }
 +
 +    assertFalse(iter.hasNext());
 +
 +    for (int i = 0; i < num; i++) {
 +      for (int j = 0; j < num; j++) {
 +        for (int k = 0; k < num; k++) {
 +          for (int l = 0; l < num; l++) {
 +            for (int ts = 0; ts < num; ts++) {
 +              Key key = nk(i, j, k, l, ts, true);
-               Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++              Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
 +
 +              assertEquals(value, nm.get(key));
 +
 +              Iterator<Entry<Key,Value>> iter2 = nm.iterator(key);
 +              assertTrue(iter2.hasNext());
 +              Entry<Key,Value> entry = iter2.next();
 +              assertEquals(key, entry.getKey());
 +              assertEquals(value, entry.getValue());
 +
 +              key = nk(i, j, k, l, ts, false);
-               value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++              value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
 +
 +              assertEquals(value, nm.get(key));
 +
 +              Iterator<Entry<Key,Value>> iter3 = nm.iterator(key);
 +              assertTrue(iter3.hasNext());
 +              Entry<Key,Value> entry2 = iter3.next();
 +              assertEquals(key, entry2.getKey());
 +              assertEquals(value, entry2.getValue());
 +            }
 +          }
 +        }
 +      }
 +    }
 +
 +    assertEquals(num * num * num * num * num * 2, nm.size());
 +  }
 +
 +  @Test
 +  public void test1() {
 +    NativeMap nm = new NativeMap();
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    assertFalse(iter.hasNext());
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test2() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerify(nm, 1, 10, 0);
 +    insertAndVerify(nm, 1, 10, 1);
 +    insertAndVerify(nm, 1, 10, 2);
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test4() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerifyExhaustive(nm, 3, 0);
 +    insertAndVerifyExhaustive(nm, 3, 1);
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test5() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerify(nm, 1, 10, 0);
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    iter.next();
 +
 +    nm.delete();
 +
 +    try {
 +      nm.put(nk(1), nv(1));
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.get(nk(1));
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.iterator();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.iterator(nk(1));
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      nm.size();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +    try {
 +      iter.next();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +
 +  }
 +
 +  @Test
 +  public void test7() {
 +    NativeMap nm = new NativeMap();
 +
 +    insertAndVerify(nm, 1, 10, 0);
 +
 +    nm.delete();
 +
 +    try {
 +      nm.delete();
 +      assertTrue(false);
 +    } catch (IllegalStateException e) {
 +
 +    }
 +  }
 +
 +  @Test
 +  public void test8() {
 +    // test verifies that native map sorts keys sharing some common prefix properly
 +
 +    NativeMap nm = new NativeMap();
 +
 +    TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
 +
 +    tm.put(new Key(new Text("fo")), new Value(new byte[] {'0'}));
 +    tm.put(new Key(new Text("foo")), new Value(new byte[] {'1'}));
 +    tm.put(new Key(new Text("foo1")), new Value(new byte[] {'2'}));
 +    tm.put(new Key(new Text("foo2")), new Value(new byte[] {'3'}));
 + 
 +    for (Entry<Key,Value> entry : tm.entrySet()) {
 +      nm.put(entry.getKey(), entry.getValue());
 +    }
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +
 +    for (Entry<Key,Value> entry : tm.entrySet()) {
 +      assertTrue(iter.hasNext());
 +      Entry<Key,Value> entry2 = iter.next();
 +
 +      assertEquals(entry.getKey(), entry2.getKey());
 +      assertEquals(entry.getValue(), entry2.getValue());
 +    }
 +
 +    assertFalse(iter.hasNext());
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test9() {
 +    NativeMap nm = new NativeMap();
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +
 +    try {
 +      iter.next();
 +      assertTrue(false);
 +    } catch (NoSuchElementException e) {
 +
 +    }
 +
 +    insertAndVerify(nm, 1, 1, 0);
 +
 +    iter = nm.iterator();
 +    iter.next();
 +
 +    try {
 +      iter.next();
 +      assertTrue(false);
 +    } catch (NoSuchElementException e) {
 +
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void test10() {
 +    int start = 1;
 +    int end = 10000;
 +
 +    NativeMap nm = new NativeMap();
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i));
 +    }
 +
 +    long mem1 = nm.getMemoryUsed();
 +
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i));
 +    }
 +
 +    long mem2 = nm.getMemoryUsed();
 +
 +    if (mem1 != mem2) {
 +      throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem2);
 +    }
 +
 +    for (int i = start; i <= end; i++) {
 +      nm.put(nk(i), nv(i));
 +    }
 +
 +    long mem3 = nm.getMemoryUsed();
 +
 +    if (mem1 != mem3) {
 +      throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem3);
 +    }
 +
 +    byte bigrow[] = new byte[1000000];
 +    byte bigvalue[] = new byte[bigrow.length];
 +
 +    for (int i = 0; i < bigrow.length; i++) {
 +      bigrow[i] = (byte) (0xff & (i % 256));
 +      bigvalue[i] = bigrow[i];
 +    }
 +
 +    nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
 +
 +    long mem4 = nm.getMemoryUsed();
 +
 +    Value val = nm.get(new Key(new Text(bigrow)));
 +    if (val == null || !val.equals(new Value(bigvalue))) {
 +      throw new RuntimeException("Did not get expected big value");
 +    }
 +
 +    nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
 +
 +    long mem5 = nm.getMemoryUsed();
 +
 +    if (mem4 != mem5) {
 +      throw new RuntimeException("Memory changed after inserting duplicate data " + mem4 + " " + mem5);
 +    }
 +
 +    val = nm.get(new Key(new Text(bigrow)));
 +    if (val == null || !val.equals(new Value(bigvalue))) {
 +      throw new RuntimeException("Did not get expected big value");
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  // random length random field
 +  private static byte[] rlrf(Random r, int maxLen) {
 +    int len = r.nextInt(maxLen);
 +
 +    byte f[] = new byte[len];
 +    r.nextBytes(f);
 +
 +    return f;
 +  }
 +
 +  @Test
 +  public void test11() {
 +    NativeMap nm = new NativeMap();
 +
 +    // insert things with varying field sizes and value sizes
 +
 +    // generate random data
 +    Random r = new Random(75);
 +
 +    ArrayList<Pair<Key,Value>> testData = new ArrayList<Pair<Key,Value>>();
 +
 +    for (int i = 0; i < 100000; i++) {
 +
 +      Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false);
 +      Value v = new Value(rlrf(r, 511));
 +
 +      testData.add(new Pair<Key,Value>(k, v));
 +    }
 +
 +    // insert unsorted data
 +    for (Pair<Key,Value> pair : testData) {
 +      nm.put(pair.getFirst(), pair.getSecond());
 +    }
 +
 +    for (int i = 0; i < 2; i++) {
 +
 +      // sort data
 +      Collections.sort(testData, new Comparator<Pair<Key,Value>>() {
 +        @Override
 +        public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
 +          return o1.getFirst().compareTo(o2.getFirst());
 +        }
 +      });
 +
 +      // verify
 +      Iterator<Entry<Key,Value>> iter1 = nm.iterator();
 +      Iterator<Pair<Key,Value>> iter2 = testData.iterator();
 +
 +      while (iter1.hasNext() && iter2.hasNext()) {
 +        Entry<Key,Value> e = iter1.next();
 +        Pair<Key,Value> p = iter2.next();
 +
 +        if (!e.getKey().equals(p.getFirst()))
 +          throw new RuntimeException("Keys not equal");
 +
 +        if (!e.getValue().equals(p.getSecond()))
 +          throw new RuntimeException("Values not equal");
 +      }
 +
 +      if (iter1.hasNext())
 +        throw new RuntimeException("Not all of native map consumed");
 +
 +      if (iter2.hasNext())
 +        throw new RuntimeException("Not all of test data consumed");
 +
 +      System.out.println("test 11 nm mem " + nm.getMemoryUsed());
 +
 +      // insert data again w/ different value
 +      Collections.shuffle(testData, r);
 +      // insert unsorted data
 +      for (Pair<Key,Value> pair : testData) {
 +        pair.getSecond().set(rlrf(r, 511));
 +        nm.put(pair.getFirst(), pair.getSecond());
 +      }
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void testBinary() {
 +    NativeMap nm = new NativeMap();
 +
 +    byte emptyBytes[] = new byte[0];
 +
 +    for (int i = 0; i < 256; i++) {
 +      for (int j = 0; j < 256; j++) {
 +        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
 +        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
 +
 +        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
 +        Value v = new Value(data);
 +
 +        nm.put(k, v);
 +      }
 +    }
 +
 +    Iterator<Entry<Key,Value>> iter = nm.iterator();
 +    for (int i = 0; i < 256; i++) {
 +      for (int j = 0; j < 256; j++) {
 +        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
 +        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
 +
 +        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
 +        Value v = new Value(data);
 +
 +        assertTrue(iter.hasNext());
 +        Entry<Key,Value> entry = iter.next();
 +
 +        assertEquals(k, entry.getKey());
 +        assertEquals(v, entry.getValue());
 +
 +      }
 +    }
 +
 +    assertFalse(iter.hasNext());
 +
 +    for (int i = 0; i < 256; i++) {
 +      for (int j = 0; j < 256; j++) {
 +        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
 +        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
 +
 +        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
 +        Value v = new Value(data);
 +
 +        Value v2 = nm.get(k);
 +
 +        assertEquals(v, v2);
 +      }
 +    }
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void testEmpty() {
 +    NativeMap nm = new NativeMap();
 +
 +    assertTrue(nm.size() == 0);
 +    assertTrue(nm.getMemoryUsed() == 0);
 +
 +    nm.delete();
 +  }
 +
 +  @Test
 +  public void testConcurrentIter() throws IOException {
 +    NativeMap nm = new NativeMap();
 +
 +    nm.put(nk(0), nv(0));
 +    nm.put(nk(1), nv(1));
 +    nm.put(nk(3), nv(3));
 +
 +    SortedKeyValueIterator<Key,Value> iter = nm.skvIterator();
 +
 +    // modify map after iter created
 +    nm.put(nk(2), nv(2));
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(0));
 +    iter.next();
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(1));
 +    iter.next();
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(2));
 +    iter.next();
 +
 +    assertTrue(iter.hasTop());
 +    assertEquals(iter.getTopKey(), nk(3));
 +    iter.next();
 +
 +    assertFalse(iter.hasTop());
 +
 +    nm.delete();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index 7e43812,0000000..189a55c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@@ -1,132 -1,0 +1,133 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.ScannerBase;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ScanIteratorIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 30;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    String tableName = getUniqueNames(1)[0];
 +    Connector c = getConnector();
 +    c.tableOperations().create(tableName);
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    for (int i = 0; i < 1000; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%06d", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(Constants.UTF8)));
-       m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(UTF_8)));
++      m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(UTF_8)));
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +
 +    Scanner scanner = c.createScanner(tableName, new Authorizations());
 +
 +    setupIter(scanner);
 +    verify(scanner, 1, 999);
 +
 +    BatchScanner bscanner = c.createBatchScanner(tableName, new Authorizations(), 3);
 +    bscanner.setRanges(Collections.singleton(new Range((Key) null, null)));
 +
 +    setupIter(bscanner);
 +    verify(bscanner, 1, 999);
 +
 +    ArrayList<Range> ranges = new ArrayList<Range>();
 +    ranges.add(new Range(new Text(String.format("%06d", 1))));
 +    ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16))));
 +    ranges.add(new Range(new Text(String.format("%06d", 20))));
 +    ranges.add(new Range(new Text(String.format("%06d", 23))));
 +    ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61))));
 +    ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504))));
 +    ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000))));
 +
 +    HashSet<Integer> got = new HashSet<Integer>();
 +    HashSet<Integer> expected = new HashSet<Integer>();
 +    for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) {
 +      expected.add(i);
 +    }
 +
 +    bscanner.setRanges(ranges);
 +
 +    for (Entry<Key,Value> entry : bscanner) {
 +      got.add(Integer.parseInt(entry.getKey().getRow().toString()));
 +    }
 +
 +    System.out.println("got : " + got);
 +
 +    if (!got.equals(expected)) {
 +      throw new Exception(got + " != " + expected);
 +    }
 +
 +    bscanner.close();
 +
 +  }
 +
 +  private void verify(Iterable<Entry<Key,Value>> scanner, int start, int finish) throws Exception {
 +
 +    int expected = start;
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (Integer.parseInt(entry.getKey().getRow().toString()) != expected) {
 +        throw new Exception("Saw unexpexted " + entry.getKey().getRow() + " " + expected);
 +      }
 +
 +      if (entry.getKey().getColumnQualifier().toString().equals("cq2")) {
 +        expected += 2;
 +      }
 +    }
 +
 +    if (expected != finish + 2) {
 +      throw new Exception("Ended at " + expected + " not " + (finish + 2));
 +    }
 +  }
 +
 +  private void setupIter(ScannerBase scanner) throws Exception {
 +    IteratorSetting dropMod = new IteratorSetting(50, "dropMod", "org.apache.accumulo.test.functional.DropModIter");
 +    dropMod.addOption("mod", "2");
 +    dropMod.addOption("drop", "0");
 +    scanner.addScanIterator(dropMod);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
index ba03d00,0000000..90b881c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
@@@ -1,242 -1,0 +1,243 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Map.Entry;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ScanRangeIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  private static final int TS_LIMIT = 1;
 +  private static final int CQ_LIMIT = 5;
 +  private static final int CF_LIMIT = 5;
 +  private static final int ROW_LIMIT = 100;
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    String[] tableNames = getUniqueNames(2);
 +    String table1 = tableNames[0];
 +    c.tableOperations().create(table1);
 +    String table2 = tableNames[1];
 +    c.tableOperations().create(table2);
 +    TreeSet<Text> splitRows = new TreeSet<Text>();
 +    int splits = 3;
 +    for (int i = (ROW_LIMIT / splits); i < ROW_LIMIT; i += (ROW_LIMIT / splits))
 +      splitRows.add(createRow(i));
 +    c.tableOperations().addSplits(table2, splitRows);
 +
 +    insertData(c, table1);
 +    scanTable(c, table1);
 +
 +    insertData(c, table2);
 +    scanTable(c, table2);
 +  }
 +
 +  private void scanTable(Connector c, String table) throws Exception {
 +    scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(1, 0, 0, 0));
 +
 +    scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +
 +    scanRange(c, table, null, null);
 +
 +    for (int i = 0; i < ROW_LIMIT; i += (ROW_LIMIT / 3)) {
 +      for (int j = 0; j < CF_LIMIT; j += (CF_LIMIT / 2)) {
 +        for (int k = 1; k < CQ_LIMIT; k += (CQ_LIMIT / 2)) {
 +          scanRange(c, table, null, new IntKey(i, j, k, 0));
 +          scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(i, j, k, 0));
 +
 +          scanRange(c, table, new IntKey(i, j, k, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +
 +          scanRange(c, table, new IntKey(i, j, k, 0), null);
 +
 +        }
 +      }
 +    }
 +
 +    for (int i = 0; i < ROW_LIMIT; i++) {
 +      scanRange(c, table, new IntKey(i, 0, 0, 0), new IntKey(i, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +
 +      if (i > 0 && i < ROW_LIMIT - 1) {
 +        scanRange(c, table, new IntKey(i - 1, 0, 0, 0), new IntKey(i + 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
 +      }
 +    }
 +
 +  }
 +
 +  private static class IntKey {
 +    private int row;
 +    private int cf;
 +    private int cq;
 +    private long ts;
 +
 +    IntKey(IntKey ik) {
 +      this.row = ik.row;
 +      this.cf = ik.cf;
 +      this.cq = ik.cq;
 +      this.ts = ik.ts;
 +    }
 +
 +    IntKey(int row, int cf, int cq, long ts) {
 +      this.row = row;
 +      this.cf = cf;
 +      this.cq = cq;
 +      this.ts = ts;
 +    }
 +
 +    Key createKey() {
 +      Text trow = createRow(row);
 +      Text tcf = createCF(cf);
 +      Text tcq = createCQ(cq);
 +
 +      return new Key(trow, tcf, tcq, ts);
 +    }
 +
 +    IntKey increment() {
 +
 +      IntKey ik = new IntKey(this);
 +
 +      ik.ts++;
 +      if (ik.ts >= TS_LIMIT) {
 +        ik.ts = 0;
 +        ik.cq++;
 +        if (ik.cq >= CQ_LIMIT) {
 +          ik.cq = 0;
 +          ik.cf++;
 +          if (ik.cf >= CF_LIMIT) {
 +            ik.cf = 0;
 +            ik.row++;
 +          }
 +        }
 +      }
 +
 +      return ik;
 +    }
 +
 +  }
 +
 +  private void scanRange(Connector c, String table, IntKey ik1, IntKey ik2) throws Exception {
 +    scanRange(c, table, ik1, false, ik2, false);
 +    scanRange(c, table, ik1, false, ik2, true);
 +    scanRange(c, table, ik1, true, ik2, false);
 +    scanRange(c, table, ik1, true, ik2, true);
 +  }
 +
 +  private void scanRange(Connector c, String table, IntKey ik1, boolean inclusive1, IntKey ik2, boolean inclusive2) throws Exception {
 +    Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
 +
 +    Key key1 = null;
 +    Key key2 = null;
 +
 +    IntKey expectedIntKey;
 +    IntKey expectedEndIntKey;
 +
 +    if (ik1 != null) {
 +      key1 = ik1.createKey();
 +      expectedIntKey = ik1;
 +
 +      if (!inclusive1) {
 +        expectedIntKey = expectedIntKey.increment();
 +      }
 +    } else {
 +      expectedIntKey = new IntKey(0, 0, 0, 0);
 +    }
 +
 +    if (ik2 != null) {
 +      key2 = ik2.createKey();
 +      expectedEndIntKey = ik2;
 +
 +      if (inclusive2) {
 +        expectedEndIntKey = expectedEndIntKey.increment();
 +      }
 +    } else {
 +      expectedEndIntKey = new IntKey(ROW_LIMIT, 0, 0, 0);
 +    }
 +
 +    Range range = new Range(key1, inclusive1, key2, inclusive2);
 +
 +    scanner.setRange(range);
 +
 +    for (Entry<Key,Value> entry : scanner) {
 +
 +      Key expectedKey = expectedIntKey.createKey();
 +      if (!expectedKey.equals(entry.getKey())) {
 +        throw new Exception(" " + expectedKey + " != " + entry.getKey());
 +      }
 +
 +      expectedIntKey = expectedIntKey.increment();
 +    }
 +
 +    if (!expectedIntKey.createKey().equals(expectedEndIntKey.createKey())) {
 +      throw new Exception(" " + expectedIntKey.createKey() + " != " + expectedEndIntKey.createKey());
 +    }
 +  }
 +
 +  private static Text createCF(int cf) {
 +    Text tcf = new Text(String.format("cf_%03d", cf));
 +    return tcf;
 +  }
 +
 +  private static Text createCQ(int cf) {
 +    Text tcf = new Text(String.format("cq_%03d", cf));
 +    return tcf;
 +  }
 +
 +  private static Text createRow(int row) {
 +    Text trow = new Text(String.format("r_%06d", row));
 +    return trow;
 +  }
 +
 +  private void insertData(Connector c, String table) throws Exception {
 +
 +    BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
 +
 +    for (int i = 0; i < ROW_LIMIT; i++) {
 +      Mutation m = new Mutation(createRow(i));
 +
 +      for (int j = 0; j < CF_LIMIT; j++) {
 +        for (int k = 0; k < CQ_LIMIT; k++) {
 +          for (int t = 0; t < TS_LIMIT; t++) {
-             m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(Constants.UTF8)));
++            m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(UTF_8)));
 +          }
 +        }
 +      }
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
index ed2f0ba,0000000..3547b68
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
@@@ -1,111 -1,0 +1,112 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ScanSessionTimeOutIT extends ConfigurableMacIT {
 +  
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_SESSION_MAXIDLE.getKey(), "3"));
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    c.tableOperations().create("abc");
 +    
 +    BatchWriter bw = c.createBatchWriter("abc", new BatchWriterConfig());
 +    
 +    for (int i = 0; i < 100000; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%08d", i)));
 +      for (int j = 0; j < 3; j++)
-         m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(Constants.UTF8)));
++        m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
 +      
 +      bw.addMutation(m);
 +    }
 +    
 +    bw.close();
 +    
 +    Scanner scanner = c.createScanner("abc", new Authorizations());
 +    scanner.setBatchSize(1000);
 +    
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +    
 +    verify(iter, 0, 200);
 +    
 +    // sleep three times the session timeout
 +    UtilWaitThread.sleep(9000);
 +    
 +    verify(iter, 200, 100000);
 +    
 +  }
 +  
 +  private void verify(Iterator<Entry<Key,Value>> iter, int start, int stop) throws Exception {
 +    for (int i = start; i < stop; i++) {
 +      
 +      Text er = new Text(String.format("%08d", i));
 +      
 +      for (int j = 0; j < 3; j++) {
 +        Entry<Key,Value> entry = iter.next();
 +        
 +        if (!entry.getKey().getRow().equals(er)) {
 +          throw new Exception("row " + entry.getKey().getRow() + " != " + er);
 +        }
 +        
 +        if (!entry.getKey().getColumnFamily().equals(new Text("cf1"))) {
 +          throw new Exception("cf " + entry.getKey().getColumnFamily() + " != cf1");
 +        }
 +        
 +        if (!entry.getKey().getColumnQualifier().equals(new Text("cq" + j))) {
 +          throw new Exception("cq " + entry.getKey().getColumnQualifier() + " != cq" + j);
 +        }
 +        
 +        if (!entry.getValue().toString().equals("" + i + "_" + j)) {
 +          throw new Exception("value " + entry.getValue() + " != " + i + "_" + j);
 +        }
 +        
 +      }
 +    }
 +    
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index ea92949,0000000..43d3a86
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@@ -1,272 -1,0 +1,273 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.ArrayList;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Writer;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.rfile.RFile;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.master.state.Assignment;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.util.FileUtil;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.tserver.TabletServer;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class SplitRecoveryIT extends ConfigurableMacIT {
 +  
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +  
 +  private KeyExtent nke(String table, String endRow, String prevEndRow) {
 +    return new KeyExtent(new Text(table), endRow == null ? null : new Text(endRow), prevEndRow == null ? null : new Text(prevEndRow));
 +  }
 +  
 +  private void run() throws Exception {
 +    String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/testLock";
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.OVERWRITE);
 +    ZooLock zl = new ZooLock(zPath);
 +    boolean gotLock = zl.tryLock(new LockWatcher() {
 +      
 +      @Override
 +      public void lostLock(LockLossReason reason) {
 +        System.exit(-1);
 +        
 +      }
 +      
 +      @Override
 +      public void unableToMonitorLockNode(Throwable e) {
 +        System.exit(-1);
 +      }
-     }, "foo".getBytes(Constants.UTF8));
++    }, "foo".getBytes(UTF_8));
 +    
 +    if (!gotLock) {
 +      System.err.println("Failed to get lock " + zPath);
 +    }
 +    
 +    // run test for a table with one tablet
 +    runSplitRecoveryTest(0, "sp", 0, zl, nke("foo0", null, null));
 +    runSplitRecoveryTest(1, "sp", 0, zl, nke("foo1", null, null));
 +    
 +    // run test for tables with two tablets, run test on first and last tablet
 +    runSplitRecoveryTest(0, "k", 0, zl, nke("foo2", "m", null), nke("foo2", null, "m"));
 +    runSplitRecoveryTest(1, "k", 0, zl, nke("foo3", "m", null), nke("foo3", null, "m"));
 +    runSplitRecoveryTest(0, "o", 1, zl, nke("foo4", "m", null), nke("foo4", null, "m"));
 +    runSplitRecoveryTest(1, "o", 1, zl, nke("foo5", "m", null), nke("foo5", null, "m"));
 +    
 +    // run test for table w/ three tablets, run test on middle tablet
 +    runSplitRecoveryTest(0, "o", 1, zl, nke("foo6", "m", null), nke("foo6", "r", "m"), nke("foo6", null, "r"));
 +    runSplitRecoveryTest(1, "o", 1, zl, nke("foo7", "m", null), nke("foo7", "r", "m"), nke("foo7", null, "r"));
 +    
 +    // run test for table w/ three tablets, run test on first
 +    runSplitRecoveryTest(0, "g", 0, zl, nke("foo8", "m", null), nke("foo8", "r", "m"), nke("foo8", null, "r"));
 +    runSplitRecoveryTest(1, "g", 0, zl, nke("foo9", "m", null), nke("foo9", "r", "m"), nke("foo9", null, "r"));
 +    
 +    // run test for table w/ three tablets, run test on last tablet
 +    runSplitRecoveryTest(0, "w", 2, zl, nke("fooa", "m", null), nke("fooa", "r", "m"), nke("fooa", null, "r"));
 +    runSplitRecoveryTest(1, "w", 2, zl, nke("foob", "m", null), nke("foob", "r", "m"), nke("foob", null, "r"));
 +  }
 +  
 +  private void runSplitRecoveryTest(int failPoint, String mr, int extentToSplit, ZooLock zl, KeyExtent... extents) throws Exception {
 +    
 +    Text midRow = new Text(mr);
 +    
 +    SortedMap<FileRef,DataFileValue> splitMapFiles = null;
 +    
 +    for (int i = 0; i < extents.length; i++) {
 +      KeyExtent extent = extents[i];
 +      
 +      String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString() + "/dir_" + i;
 +      MetadataTableUtil.addTablet(extent, tdir, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID, zl);
 +      SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>();
 +      mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i));
 +      
 +      if (i == extentToSplit) {
 +        splitMapFiles = mapFiles;
 +      }
 +      int tid = 0;
 +      TransactionWatcher.ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
 +      MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles, "L0", SystemCredentials.get(), zl);
 +    }
 +    
 +    KeyExtent extent = extents[extentToSplit];
 +    
 +    KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
 +    KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
 +    
 +    splitPartiallyAndRecover(extent, high, low, .4, splitMapFiles, midRow, "localhost:1234", failPoint, zl);
 +  }
 +  
 +  private void splitPartiallyAndRecover(KeyExtent extent, KeyExtent high, KeyExtent low, double splitRatio, SortedMap<FileRef,DataFileValue> mapFiles,
 +      Text midRow, String location, int steps, ZooLock zl) throws Exception {
 +    
 +    SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>();
 +    SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
 +    List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
 +    
 +    MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, new HashMap<FileRef,FileUtil.FileInfo>(), mapFiles, lowDatafileSizes,
 +        highDatafileSizes, highDatafilesToRemove);
 +    
 +    MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, SystemCredentials.get(), zl);
 +    TServerInstance instance = new TServerInstance(location, zl.getSessionId());
 +    Writer writer = new Writer(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID);
 +    Assignment assignment = new Assignment(high, instance);
 +    Mutation m = new Mutation(assignment.tablet.getMetadataEntry());
 +    m.put(TabletsSection.FutureLocationColumnFamily.NAME, assignment.server.asColumnQualifier(), assignment.server.asMutationValue());
 +    writer.update(m);
 +    
 +    if (steps >= 1) {
 +      Map<FileRef,Long> bulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), extent);
 +      MasterMetadataUtil.addNewTablet(low, "/lowDir", instance, lowDatafileSizes, bulkFiles, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID + "0", -1l,
 +          -1l, zl);
 +    }
 +    if (steps >= 2)
 +      MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SystemCredentials.get(), zl);
 +    
 +    TabletServer.verifyTabletInformation(high, instance, null, "127.0.0.1:0", zl);
 +    
 +    if (steps >= 1) {
 +      ensureTabletHasNoUnexpectedMetadataEntries(low, lowDatafileSizes);
 +      ensureTabletHasNoUnexpectedMetadataEntries(high, highDatafileSizes);
 +      
 +      Map<FileRef,Long> lowBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), low);
 +      Map<FileRef,Long> highBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), high);
 +      
 +      if (!lowBulkFiles.equals(highBulkFiles)) {
 +        throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
 +      }
 +      
 +      if (lowBulkFiles.size() == 0) {
 +        throw new Exception(" no bulk files " + low);
 +      }
 +    } else {
 +      ensureTabletHasNoUnexpectedMetadataEntries(extent, mapFiles);
 +    }
 +  }
 +  
 +  private void ensureTabletHasNoUnexpectedMetadataEntries(KeyExtent extent, SortedMap<FileRef,DataFileValue> expectedMapFiles) throws Exception {
 +    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(extent.toMetadataRange());
 +    
 +    HashSet<ColumnFQ> expectedColumns = new HashSet<ColumnFQ>();
 +    expectedColumns.add(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN);
 +    expectedColumns.add(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
 +    expectedColumns.add(TabletsSection.ServerColumnFamily.TIME_COLUMN);
 +    expectedColumns.add(TabletsSection.ServerColumnFamily.LOCK_COLUMN);
 +    
 +    HashSet<Text> expectedColumnFamilies = new HashSet<Text>();
 +    expectedColumnFamilies.add(DataFileColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.FutureLocationColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.CurrentLocationColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.LastLocationColumnFamily.NAME);
 +    expectedColumnFamilies.add(TabletsSection.BulkFileColumnFamily.NAME);
 +    
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +    while (iter.hasNext()) {
 +      Key key = iter.next().getKey();
 +      
 +      if (!key.getRow().equals(extent.getMetadataEntry())) {
 +        throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
 +      }
 +      
 +      if (expectedColumnFamilies.contains(key.getColumnFamily())) {
 +        continue;
 +      }
 +      
 +      if (expectedColumns.remove(new ColumnFQ(key))) {
 +        continue;
 +      }
 +      
 +      throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
 +    }
 +    System.out.println("expectedColumns " + expectedColumns);
 +    if (expectedColumns.size() > 1 || (expectedColumns.size() == 1)) {
 +      throw new Exception("Not all expected columns seen " + extent + " " + expectedColumns);
 +    }
 +    
 +    SortedMap<FileRef,DataFileValue> fixedMapFiles = MetadataTableUtil.getDataFileSizes(extent, SystemCredentials.get());
 +    verifySame(expectedMapFiles, fixedMapFiles);
 +  }
 +  
 +  private void verifySame(SortedMap<FileRef,DataFileValue> datafileSizes, SortedMap<FileRef,DataFileValue> fixedDatafileSizes) throws Exception {
 +    
 +    if (!datafileSizes.keySet().containsAll(fixedDatafileSizes.keySet()) || !fixedDatafileSizes.keySet().containsAll(datafileSizes.keySet())) {
 +      throw new Exception("Key sets not the same " + datafileSizes.keySet() + " !=  " + fixedDatafileSizes.keySet());
 +    }
 +    
 +    for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
 +      DataFileValue dfv = entry.getValue();
 +      DataFileValue otherDfv = fixedDatafileSizes.get(entry.getKey());
 +      
 +      if (!dfv.equals(otherDfv)) {
 +        throw new Exception(entry.getKey() + " dfv not equal  " + dfv + "  " + otherDfv);
 +      }
 +    }
 +  }
 +
 +  
 +  public static void main(String[] args) throws Exception {
 +    new SplitRecoveryIT().run();
 +  }
 +  
 +  @Test
 +  public void test() throws Exception {
 +    assertEquals(0, exec(SplitRecoveryIT.class).waitFor());
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
index b74399d,0000000..fccc79f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
@@@ -1,101 -1,0 +1,101 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.minicluster.MemoryUnit;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class TabletIT extends ConfigurableMacIT {
 +
 +  private static final int N = 1000;
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    Map<String,String> siteConfig = new HashMap<String,String>();
 +    siteConfig.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "200");
 +    siteConfig.put(Property.TSERV_MAXMEM.getKey(), "128M");
 +    cfg.setDefaultMemory(256, MemoryUnit.MEGABYTE);
 +    cfg.setSiteConfig(siteConfig);
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void createTableTest() throws Exception {
 +    String tableName = getUniqueNames(1)[0];
 +    createTableTest(tableName, false);
 +    createTableTest(tableName, true);
 +  }
 +
 +  public void createTableTest(String tableName, boolean readOnly) throws Exception {
 +    // create the test table within accumulo
 +    Connector connector = getConnector();
 +
 +    if (!readOnly) {
 +      TreeSet<Text> keys = new TreeSet<Text>();
 +      for (int i = N / 100; i < N; i += N / 100) {
 +        keys.add(new Text(String.format("%05d", i)));
 +      }
 +
 +      // presplit
 +      connector.tableOperations().create(tableName);
 +      connector.tableOperations().addSplits(tableName, keys);
 +      BatchWriter b = connector.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +      // populate
 +      for (int i = 0; i < N; i++) {
 +        Mutation m = new Mutation(new Text(String.format("%05d", i)));
-         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
++        m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
 +        b.addMutation(m);
 +      }
 +      b.close();
 +    }
 +
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +    int count = 0;
 +    for (Entry<Key,Value> elt : scanner) {
 +      String expected = String.format("%05d", count);
 +      assert (elt.getKey().getRow().toString().equals(expected));
 +      count++;
 +    }
 +    assertEquals(N, count);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
index 1dd0bf1,0000000..f2460cf
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@@ -1,303 -1,0 +1,304 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.util.ByteArraySet;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class VisibilityIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    String[] tableNames = getUniqueNames(2);
 +    String table = tableNames[0];
 +    c.tableOperations().create(table);
 +    String table2 = tableNames[1];
 +    c.tableOperations().create(table2);
 +    c.tableOperations().setProperty(table2, Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "DEFLABEL");
 +
 +    insertData(c, table);
 +    queryData(c, table);
 +    deleteData(c, table);
 +
 +    insertDefaultData(c, table2);
 +    queryDefaultData(c, table2);
 +
 +  }
 +
 +  private static SortedSet<String> nss(String... labels) {
 +    TreeSet<String> ts = new TreeSet<String>();
 +
 +    for (String s : labels) {
 +      ts.add(s);
 +    }
 +
 +    return ts;
 +  }
 +
 +  private void mput(Mutation m, String cf, String cq, String cv, String val) {
-     ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
-     m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(Constants.UTF8)));
++    ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
++    m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(UTF_8)));
 +  }
 +
 +  private void mputDelete(Mutation m, String cf, String cq, String cv) {
-     ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
++    ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
 +    m.putDelete(new Text(cf), new Text(cq), le);
 +  }
 +
 +  private void insertData(Connector c, String tableName) throws Exception {
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m1 = new Mutation(new Text("row1"));
 +
 +    mput(m1, "cf1", "cq1", "", "v1");
 +    mput(m1, "cf1", "cq1", "A", "v2");
 +    mput(m1, "cf1", "cq1", "B", "v3");
 +    mput(m1, "cf1", "cq1", "A&B", "v4");
 +    mput(m1, "cf1", "cq1", "A&(L|M)", "v5");
 +    mput(m1, "cf1", "cq1", "B&(L|M)", "v6");
 +    mput(m1, "cf1", "cq1", "A&B&(L|M)", "v7");
 +    mput(m1, "cf1", "cq1", "A&B&(L)", "v8");
 +    mput(m1, "cf1", "cq1", "A&FOO", "v9");
 +    mput(m1, "cf1", "cq1", "A&FOO&(L|M)", "v10");
 +    mput(m1, "cf1", "cq1", "FOO", "v11");
 +    mput(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)", "v12");
 +    mput(m1, "cf1", "cq1", "A&B&(L|M|FOO)", "v13");
 +
 +    bw.addMutation(m1);
 +    bw.close();
 +  }
 +
 +  private void deleteData(Connector c, String tableName) throws Exception {
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m1 = new Mutation(new Text("row1"));
 +
 +    mputDelete(m1, "cf1", "cq1", "");
 +    mputDelete(m1, "cf1", "cq1", "A");
 +    mputDelete(m1, "cf1", "cq1", "A&B");
 +    mputDelete(m1, "cf1", "cq1", "B&(L|M)");
 +    mputDelete(m1, "cf1", "cq1", "A&B&(L)");
 +    mputDelete(m1, "cf1", "cq1", "A&FOO&(L|M)");
 +    mputDelete(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)");
 +    mputDelete(m1, "cf1", "cq1", "FOO&A"); // should not delete anything
 +
 +    bw.addMutation(m1);
 +    bw.close();
 +
 +    Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
 +
 +    expected.put(nss("A", "L"), nss("v5"));
 +    expected.put(nss("A", "M"), nss("v5"));
 +    expected.put(nss("B"), nss("v3"));
 +    expected.put(nss("Z"), nss());
 +    expected.put(nss("A", "B", "L"), nss("v7", "v13"));
 +    expected.put(nss("A", "B", "M"), nss("v7", "v13"));
 +    expected.put(nss("A", "B", "FOO"), nss("v13"));
 +    expected.put(nss("FOO"), nss("v11"));
 +    expected.put(nss("A", "FOO"), nss("v9"));
 +
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
 +  }
 +
 +  private void insertDefaultData(Connector c, String tableName) throws Exception {
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m1 = new Mutation(new Text("row1"));
 +
 +    mput(m1, "cf1", "cq1", "BASE", "v1");
 +    mput(m1, "cf1", "cq2", "DEFLABEL", "v2");
 +    mput(m1, "cf1", "cq3", "", "v3");
 +
 +    bw.addMutation(m1);
 +    bw.close();
 +  }
 +
 +  private static void uniqueCombos(List<Set<String>> all, Set<String> prefix, Set<String> suffix) {
 +
 +    all.add(prefix);
 +
 +    TreeSet<String> ss = new TreeSet<String>(suffix);
 +
 +    for (String s : suffix) {
 +      TreeSet<String> ps = new TreeSet<String>(prefix);
 +      ps.add(s);
 +      ss.remove(s);
 +
 +      uniqueCombos(all, ps, ss);
 +    }
 +  }
 +
 +  private void queryData(Connector c, String tableName) throws Exception {
 +    Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
 +    expected.put(nss(), nss("v1"));
 +    expected.put(nss("A"), nss("v2"));
 +    expected.put(nss("A", "L"), nss("v5"));
 +    expected.put(nss("A", "M"), nss("v5"));
 +    expected.put(nss("B"), nss("v3"));
 +    expected.put(nss("B", "L"), nss("v6"));
 +    expected.put(nss("B", "M"), nss("v6"));
 +    expected.put(nss("Z"), nss());
 +    expected.put(nss("A", "B"), nss("v4"));
 +    expected.put(nss("A", "B", "L"), nss("v7", "v8", "v13"));
 +    expected.put(nss("A", "B", "M"), nss("v7", "v13"));
 +    expected.put(nss("A", "B", "FOO"), nss("v13"));
 +    expected.put(nss("FOO"), nss("v11"));
 +    expected.put(nss("A", "FOO"), nss("v9"));
 +    expected.put(nss("A", "FOO", "L"), nss("v10", "v12"));
 +    expected.put(nss("A", "FOO", "M"), nss("v10", "v12"));
 +    expected.put(nss("B", "FOO", "L"), nss("v12"));
 +    expected.put(nss("B", "FOO", "M"), nss("v12"));
 +
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "L", "M", "Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("Z"), expected);
 +    queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected);
 +  }
 +
 +  private void queryData(Connector c, String tableName, Set<String> allAuths, Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws Exception {
 +
 +    c.securityOperations().changeUserAuthorizations("root", new Authorizations(nbas(userAuths)));
 +
 +    ArrayList<Set<String>> combos = new ArrayList<Set<String>>();
 +    uniqueCombos(combos, nss(), allAuths);
 +
 +    for (Set<String> set1 : combos) {
 +      Set<String> e = new TreeSet<String>();
 +      for (Set<String> set2 : combos) {
 +
 +        set2 = new HashSet<String>(set2);
 +        set2.retainAll(userAuths);
 +
 +        if (set1.containsAll(set2) && expected.containsKey(set2)) {
 +          e.addAll(expected.get(set2));
 +        }
 +      }
 +
 +      set1.retainAll(userAuths);
 +      verify(c, tableName, set1, e);
 +    }
 +
 +  }
 +
 +  private void queryDefaultData(Connector c, String tableName) throws Exception {
 +    Scanner scanner;
 +
 +    // should return no records
 +    c.securityOperations().changeUserAuthorizations("root", new Authorizations("BASE", "DEFLABEL"));
 +    scanner = getConnector().createScanner(tableName, new Authorizations());
 +    verifyDefault(scanner, 0);
 +
 +    // should return one record
 +    scanner = getConnector().createScanner(tableName, new Authorizations("BASE"));
 +    verifyDefault(scanner, 1);
 +
 +    // should return all three records
 +    scanner = getConnector().createScanner(tableName, new Authorizations("BASE", "DEFLABEL"));
 +    verifyDefault(scanner, 3);
 +  }
 +
 +  private void verifyDefault(Scanner scanner, int expectedCount) throws Exception {
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner)
 +      --expectedCount;
 +    if (expectedCount != 0)
 +      throw new Exception(" expected count !=0 " + expectedCount);
 +  }
 +
 +  private void verify(Connector c, String tableName, Set<String> auths, Set<String> expectedValues) throws Exception {
 +    ByteArraySet bas = nbas(auths);
 +
 +    try {
 +      verify(c, tableName, bas, expectedValues.toArray(new String[0]));
 +    } catch (Exception e) {
 +      throw new Exception("Verification failed auths=" + auths + " exp=" + expectedValues, e);
 +    }
 +  }
 +
 +  private ByteArraySet nbas(Set<String> auths) {
 +    ByteArraySet bas = new ByteArraySet();
 +    for (String auth : auths) {
-       bas.add(auth.getBytes(Constants.UTF8));
++      bas.add(auth.getBytes(UTF_8));
 +    }
 +    return bas;
 +  }
 +
 +  private void verify(Connector c, String tableName, ByteArraySet nss, String... expected) throws Exception {
 +    Scanner scanner = c.createScanner(tableName, new Authorizations(nss));
 +    verify(scanner.iterator(), expected);
 +
 +    BatchScanner bs = getConnector().createBatchScanner(tableName, new Authorizations(nss), 3);
 +    bs.setRanges(Collections.singleton(new Range()));
 +    verify(bs.iterator(), expected);
 +    bs.close();
 +  }
 +
 +  private void verify(Iterator<Entry<Key,Value>> iter, String... expected) throws Exception {
 +    HashSet<String> valuesSeen = new HashSet<String>();
 +
 +    while (iter.hasNext()) {
 +      Entry<Key,Value> entry = iter.next();
 +      if (valuesSeen.contains(entry.getValue().toString())) {
 +        throw new Exception("Value seen twice");
 +      }
 +      valuesSeen.add(entry.getValue().toString());
 +    }
 +
 +    for (String ev : expected) {
 +      if (!valuesSeen.remove(ev)) {
 +        throw new Exception("Did not see expected value " + ev);
 +      }
 +    }
 +
 +    if (valuesSeen.size() != 0) {
 +      throw new Exception("Saw more values than expected " + valuesSeen);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------


[09/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
index 1226806,0000000..10cf224
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
@@@ -1,617 -1,0 +1,619 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.io.OutputStreamWriter;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.concurrent.Callable;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Future;
 +import java.util.concurrent.ThreadPoolExecutor;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.ServerClient;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.SimpleThreadPool;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
 +import org.apache.accumulo.trace.instrument.TraceExecutorService;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.MapFile;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +
 +/*
 + * Bulk import makes requests of tablet servers, and those requests can take a
 + * long time. Our communications to the tablet server may fail, so we won't know
 + * the status of the request. The master will repeat failed requests so now
 + * there are multiple requests to the tablet server. The tablet server will not
 + * execute the request multiple times, so long as the marker it wrote in the
 + * metadata table stays there. The master needs to know when all requests have
 + * finished so it can remove the markers. Did it start? Did it finish? We can see
 + * that *a* request completed by seeing the flag written into the metadata
 + * table, but we won't know if some other rogue thread is still waiting to start
 + * a thread and repeat the operation.
 + * 
 + * The master can ask the tablet server if it has any requests still running.
 + * Except the tablet server might have some thread about to start a request, but
 + * before it has made any bookkeeping about the request. To prevent problems
 + * like this, an Arbitrator is used. Before starting any new request, the tablet
 + * server checks the Arbitrator to see if the request is still valid.
 + * 
 + */
 +
 +public class BulkImport extends MasterRepo {
 +  public static final String FAILURES_TXT = "failures.txt";
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private static final Logger log = Logger.getLogger(BulkImport.class);
 +
 +  private String tableId;
 +  private String sourceDir;
 +  private String errorDir;
 +  private boolean setTime;
 +
 +  public BulkImport(String tableId, String sourceDir, String errorDir, boolean setTime) {
 +    this.tableId = tableId;
 +    this.sourceDir = sourceDir;
 +    this.errorDir = errorDir;
 +    this.setTime = setTime;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    if (!Utils.getReadLock(tableId, tid).tryLock())
 +      return 100;
 +
 +    Instance instance = HdfsZooInstance.getInstance();
 +    Tables.clearCache(instance);
 +    if (Tables.getTableState(instance, tableId) == TableState.ONLINE) {
 +      long reserve1, reserve2;
 +      reserve1 = reserve2 = Utils.reserveHdfsDirectory(sourceDir, tid);
 +      if (reserve1 == 0)
 +        reserve2 = Utils.reserveHdfsDirectory(errorDir, tid);
 +      return reserve2;
 +    } else {
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.OFFLINE, null);
 +    }
 +  }
 +
 +  @Override
 +  //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
 +  @SuppressWarnings("deprecation")
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    log.debug(" tid " + tid + " sourceDir " + sourceDir);
 +
 +    Utils.getReadLock(tableId, tid).lock();
 +
 +    // check that the error directory exists and is empty
 +    VolumeManager fs = master.getFileSystem();
 +
 +    Path errorPath = new Path(errorDir);
 +    FileStatus errorStatus = null;
 +    try {
 +      errorStatus = fs.getFileStatus(errorPath);
 +    } catch (FileNotFoundException ex) {
 +      // ignored
 +    }
 +    if (errorStatus == null)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
 +          + " does not exist");
 +    if (!errorStatus.isDir())
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
 +          + " is not a directory");
 +    if (fs.listStatus(errorPath).length != 0)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
 +          + " is not empty");
 +
 +    ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
 +
 +    // move the files into the directory
 +    try {
 +      String bulkDir = prepareBulkImport(fs, sourceDir, tableId);
 +      log.debug(" tid " + tid + " bulkDir " + bulkDir);
 +      return new LoadFiles(tableId, sourceDir, bulkDir, errorDir, setTime);
 +    } catch (IOException ex) {
 +      log.error("error preparing the bulk import directory", ex);
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_INPUT_DIRECTORY, sourceDir + ": "
 +          + ex);
 +    }
 +  }
 +
 +  private Path createNewBulkDir(VolumeManager fs, String tableId) throws IOException {
 +    Path tempPath = fs.matchingFileSystem(new Path(sourceDir), ServerConstants.getTablesDirs());
 +    if (tempPath == null)
 +      throw new IllegalStateException(sourceDir + " is not in a known namespace");
 +
 +    String tableDir = tempPath.toString();
 +    if (tableDir == null)
 +      throw new IllegalStateException(sourceDir + " is not in a known namespace");
 +    Path directory = new Path(tableDir + "/" + tableId);
 +    fs.mkdirs(directory);
 +
 +    // only one should be able to create the lock file
 +    // the purpose of the lock file is to avoid a race
 +    // condition between the call to fs.exists() and
 +    // fs.mkdirs()... if only hadoop had a mkdir() function
 +    // that failed when the dir existed
 +
 +    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
 +    while (true) {
 +      Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
 +      if (fs.exists(newBulkDir)) // sanity check
 +        throw new IllegalStateException("Dir exist when it should not " + newBulkDir);
 +      if (fs.mkdirs(newBulkDir))
 +        return newBulkDir;
 +      log.warn("Failed to create " + newBulkDir + " for unknown reason");
 +
 +      UtilWaitThread.sleep(3000);
 +    }
 +  }
 +
 +  //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
 +  @SuppressWarnings("deprecation")
 +  private String prepareBulkImport(VolumeManager fs, String dir, String tableId) throws IOException {
 +    Path bulkDir = createNewBulkDir(fs, tableId);
 +
 +    MetadataTableUtil.addBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
 +
 +    Path dirPath = new Path(dir);
 +    FileStatus[] mapFiles = fs.listStatus(dirPath);
 +
 +    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
 +    for (FileStatus fileStatus : mapFiles) {
 +      String sa[] = fileStatus.getPath().getName().split("\\.");
 +      String extension = "";
 +      if (sa.length > 1) {
 +        extension = sa[sa.length - 1];
 +
 +        if (!FileOperations.getValidExtensions().contains(extension)) {
 +          log.warn(fileStatus.getPath() + " does not have a valid extension, ignoring");
 +          continue;
 +        }
 +      } else {
 +        // assume it is a map file
 +        extension = Constants.MAPFILE_EXTENSION;
 +      }
 +
 +      if (extension.equals(Constants.MAPFILE_EXTENSION)) {
 +        if (!fileStatus.isDir()) {
 +          log.warn(fileStatus.getPath() + " is not a map file, ignoring");
 +          continue;
 +        }
 +
 +        if (fileStatus.getPath().getName().equals("_logs")) {
 +          log.info(fileStatus.getPath() + " is probably a log directory from a map/reduce task, skipping");
 +          continue;
 +        }
 +        try {
 +          FileStatus dataStatus = fs.getFileStatus(new Path(fileStatus.getPath(), MapFile.DATA_FILE_NAME));
 +          if (dataStatus.isDir()) {
 +            log.warn(fileStatus.getPath() + " is not a map file, ignoring");
 +            continue;
 +          }
 +        } catch (FileNotFoundException fnfe) {
 +          log.warn(fileStatus.getPath() + " is not a map file, ignoring");
 +          continue;
 +        }
 +      }
 +
 +      String newName = "I" + namer.getNextName() + "." + extension;
 +      Path newPath = new Path(bulkDir, newName);
 +      try {
 +        fs.rename(fileStatus.getPath(), newPath);
 +        log.debug("Moved " + fileStatus.getPath() + " to " + newPath);
 +      } catch (IOException E1) {
 +        log.error("Could not move: " + fileStatus.getPath().toString() + " " + E1.getMessage());
 +      }
 +    }
 +    return bulkDir.toString();
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    // unreserve source/error directories
 +    Utils.unreserveHdfsDirectory(sourceDir, tid);
 +    Utils.unreserveHdfsDirectory(errorDir, tid);
 +    Utils.getReadLock(tableId, tid).unlock();
 +  }
 +}
 +
 +class CleanUpBulkImport extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private static final Logger log = Logger.getLogger(CleanUpBulkImport.class);
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String error;
 +
 +  public CleanUpBulkImport(String tableId, String source, String bulk, String error) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.error = error;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    log.debug("removing the bulk processing flag file in " + bulk);
 +    Path bulkDir = new Path(bulk);
 +    MetadataTableUtil.removeBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
 +    MetadataTableUtil.addDeleteEntry(tableId, bulkDir.toString());
 +    log.debug("removing the metadata table markers for loaded files");
 +    Connector conn = master.getConnector();
 +    MetadataTableUtil.removeBulkLoadEntries(conn, tableId, tid);
 +    log.debug("releasing HDFS reservations for " + source + " and " + error);
 +    Utils.unreserveHdfsDirectory(source, tid);
 +    Utils.unreserveHdfsDirectory(error, tid);
 +    Utils.getReadLock(tableId, tid).unlock();
 +    log.debug("completing bulk import transaction " + tid);
 +    ZooArbitrator.cleanup(Constants.BULK_ARBITRATOR_TYPE, tid);
 +    return null;
 +  }
 +}
 +
 +class CompleteBulkImport extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String error;
 +
 +  public CompleteBulkImport(String tableId, String source, String bulk, String error) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.error = error;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    ZooArbitrator.stop(Constants.BULK_ARBITRATOR_TYPE, tid);
 +    return new CopyFailed(tableId, source, bulk, error);
 +  }
 +}
 +
 +class CopyFailed extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String error;
 +
 +  public CopyFailed(String tableId, String source, String bulk, String error) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.error = error;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    Set<TServerInstance> finished = new HashSet<TServerInstance>();
 +    Set<TServerInstance> running = master.onlineTabletServers();
 +    for (TServerInstance server : running) {
 +      try {
 +        TServerConnection client = master.getConnection(server);
 +        if (client != null && !client.isActive(tid))
 +          finished.add(server);
 +      } catch (TException ex) {
 +        log.info("Ignoring error trying to check on tid " + tid + " from server " + server + ": " + ex);
 +      }
 +    }
 +    if (finished.containsAll(running))
 +      return 0;
 +    return 500;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    // This needs to execute after the arbiter is stopped
 +
 +    VolumeManager fs = master.getFileSystem();
 +
 +    if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
 +      return new CleanUpBulkImport(tableId, source, bulk, error);
 +
 +    HashMap<FileRef,String> failures = new HashMap<FileRef,String>();
 +    HashMap<FileRef,String> loadedFailures = new HashMap<FileRef,String>();
 +
 +    FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
-     BufferedReader in = new BufferedReader(new InputStreamReader(failFile, Constants.UTF8));
++    BufferedReader in = new BufferedReader(new InputStreamReader(failFile, UTF_8));
 +    try {
 +      String line = null;
 +      while ((line = in.readLine()) != null) {
 +        Path path = new Path(line);
 +        if (!fs.exists(new Path(error, path.getName())))
 +          failures.put(new FileRef(line, path), line);
 +      }
 +    } finally {
 +      failFile.close();
 +    }
 +
 +    /*
 +     * I thought I could move files that have no file references in the table. However its possible a clone references a file. Therefore only move files that
 +     * have no loaded markers.
 +     */
 +
 +    // determine which failed files were loaded
 +    Connector conn = master.getConnector();
 +    Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +
 +    for (Entry<Key,Value> entry : mscanner) {
 +      if (Long.parseLong(entry.getValue().toString()) == tid) {
 +        FileRef loadedFile = new FileRef(fs, entry.getKey());
 +        String absPath = failures.remove(loadedFile);
 +        if (absPath != null) {
 +          loadedFailures.put(loadedFile, absPath);
 +        }
 +      }
 +    }
 +
 +    // move failed files that were not loaded
 +    for (String failure : failures.values()) {
 +      Path orig = new Path(failure);
 +      Path dest = new Path(error, orig.getName());
 +      fs.rename(orig, dest);
 +      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
 +    }
 +
 +    if (loadedFailures.size() > 0) {
 +      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
 +          + Constants.ZBULK_FAILED_COPYQ);
 +
 +      HashSet<String> workIds = new HashSet<String>();
 +
 +      for (String failure : loadedFailures.values()) {
 +        Path orig = new Path(failure);
 +        Path dest = new Path(error, orig.getName());
 +
 +        if (fs.exists(dest))
 +          continue;
 +
-         bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
++        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8));
 +        workIds.add(orig.getName());
 +        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
 +      }
 +
 +      bifCopyQueue.waitUntilDone(workIds);
 +    }
 +
 +    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
 +    return new CleanUpBulkImport(tableId, source, bulk, error);
 +  }
 +
 +}
 +
 +class LoadFiles extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private static ExecutorService threadPool = null;
 +  private static final Logger log = Logger.getLogger(BulkImport.class);
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String errorDir;
 +  private boolean setTime;
 +
 +  public LoadFiles(String tableId, String source, String bulk, String errorDir, boolean setTime) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.errorDir = errorDir;
 +    this.setTime = setTime;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    if (master.onlineTabletServers().size() == 0)
 +      return 500;
 +    return 0;
 +  }
 +
 +  private static synchronized ExecutorService getThreadPool(Master master) {
 +    if (threadPool == null) {
 +      int threadPoolSize = master.getSystemConfiguration().getCount(Property.MASTER_BULK_THREADPOOL_SIZE);
 +      ThreadPoolExecutor pool = new SimpleThreadPool(threadPoolSize, "bulk import");
 +      pool.allowCoreThreadTimeOut(true);
 +      threadPool = new TraceExecutorService(pool);
 +    }
 +    return threadPool;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(final long tid, final Master master) throws Exception {
 +    ExecutorService executor = getThreadPool(master);
 +    final SiteConfiguration conf = ServerConfiguration.getSiteConfiguration();
 +    VolumeManager fs = master.getFileSystem();
 +    List<FileStatus> files = new ArrayList<FileStatus>();
 +    for (FileStatus entry : fs.listStatus(new Path(bulk))) {
 +      files.add(entry);
 +    }
 +    log.debug("tid " + tid + " importing " + files.size() + " files");
 +
 +    Path writable = new Path(this.errorDir, ".iswritable");
 +    if (!fs.createNewFile(writable)) {
 +      // Maybe this is a re-try... clear the flag and try again
 +      fs.delete(writable);
 +      if (!fs.createNewFile(writable))
 +        throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY,
 +            "Unable to write to " + this.errorDir);
 +    }
 +    fs.delete(writable);
 +
 +    final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<String>());
 +    for (FileStatus f : files)
 +      filesToLoad.add(f.getPath().toString());
 +
 +    final int RETRIES = Math.max(1, conf.getCount(Property.MASTER_BULK_RETRIES));
 +    for (int attempt = 0; attempt < RETRIES && filesToLoad.size() > 0; attempt++) {
 +      List<Future<List<String>>> results = new ArrayList<Future<List<String>>>();
 +
 +      if (master.onlineTabletServers().size() == 0)
 +        log.warn("There are no tablet server to process bulk import, waiting (tid = " + tid + ")");
 +
 +      while (master.onlineTabletServers().size() == 0) {
 +        UtilWaitThread.sleep(500);
 +      }
 +
 +      // Use the threadpool to assign files one-at-a-time to the server
 +      final List<String> loaded = Collections.synchronizedList(new ArrayList<String>());
 +      for (final String file : filesToLoad) {
 +        results.add(executor.submit(new Callable<List<String>>() {
 +          @Override
 +          public List<String> call() {
 +            List<String> failures = new ArrayList<String>();
 +            ClientService.Client client = null;
 +            String server = null;
 +            try {
 +              // get a connection to a random tablet server, do not prefer cached connections because
 +              // this is running on the master and there are lots of connections to tablet servers
 +              // serving the metadata tablets
 +              long timeInMillis = master.getConfiguration().getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
 +              Pair<String,Client> pair = ServerClient.getConnection(master.getInstance(), false, timeInMillis);
 +              client = pair.getSecond();
 +              server = pair.getFirst();
 +              List<String> attempt = Collections.singletonList(file);
 +              log.debug("Asking " + pair.getFirst() + " to bulk import " + file);
 +              List<String> fail = client.bulkImportFiles(Tracer.traceInfo(), SystemCredentials.get().toThrift(master.getInstance()), tid, tableId, attempt,
 +                  errorDir, setTime);
 +              if (fail.isEmpty()) {
 +                loaded.add(file);
 +              } else {
 +                failures.addAll(fail);
 +              }
 +            } catch (Exception ex) {
 +              log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
 +            } finally {
 +              ServerClient.close(client);
 +            }
 +            return failures;
 +          }
 +        }));
 +      }
 +      Set<String> failures = new HashSet<String>();
 +      for (Future<List<String>> f : results)
 +        failures.addAll(f.get());
 +      filesToLoad.removeAll(loaded);
 +      if (filesToLoad.size() > 0) {
 +        log.debug("tid " + tid + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
 +        UtilWaitThread.sleep(100);
 +      }
 +    }
 +
 +    FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
-     BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, Constants.UTF8));
++    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8));
 +    try {
 +      for (String f : filesToLoad) {
 +        out.write(f);
 +        out.write("\n");
 +      }
 +    } finally {
 +      out.close();
 +    }
 +
 +    // return the next step, which will perform cleanup
 +    return new CompleteBulkImport(tableId, source, bulk, errorDir);
 +  }
 +
 +  static String sampleList(Collection<?> potentiallyLongList, int max) {
 +    StringBuffer result = new StringBuffer();
 +    result.append("[");
 +    int i = 0;
 +    for (Object obj : potentiallyLongList) {
 +      result.append(obj);
 +      if (i >= max) {
 +        result.append("...");
 +        break;
 +      } else {
 +        result.append(", ");
 +      }
 +      i++;
 +    }
 +    if (i < max)
 +      result.delete(result.length() - 2, result.length());
 +    result.append("]");
 +    return result.toString();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
index ebceaa0,0000000..fd2441c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
@@@ -1,105 -1,0 +1,107 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +class FinishCancelCompaction extends MasterRepo {
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +
 +  public FinishCancelCompaction(String tableId) {
 +    this.tableId = tableId;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +    Utils.getReadLock(tableId, tid).unlock();
 +    return null;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +
 +  }
 +}
 +
 +/**
 + *
 + */
 +public class CancelCompactions extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +  private String namespaceId;
 +
 +  public CancelCompactions(String tableId) {
 +    this.tableId = tableId;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT_CANCEL)
 +        + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT_CANCEL);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +    String zCompactID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
 +    String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    byte[] currentValue = zoo.getData(zCompactID, null);
 +
-     String cvs = new String(currentValue, Constants.UTF8);
++    String cvs = new String(currentValue, UTF_8);
 +    String[] tokens = cvs.split(",");
 +    final long flushID = Long.parseLong(tokens[0]);
 +
 +    zoo.mutate(zCancelID, null, null, new Mutator() {
 +      @Override
 +      public byte[] mutate(byte[] currentValue) throws Exception {
-         long cid = Long.parseLong(new String(currentValue, Constants.UTF8));
++        long cid = Long.parseLong(new String(currentValue, UTF_8));
 +
 +        if (cid < flushID)
-           return Long.toString(flushID).getBytes(Constants.UTF8);
++          return Long.toString(flushID).getBytes(UTF_8);
 +        else
-           return Long.toString(cid).getBytes(Constants.UTF8);
++          return Long.toString(cid).getBytes(UTF_8);
 +      }
 +    });
 +
 +    return new FinishCancelCompaction(tableId);
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    Utils.unreserveNamespace(namespaceId, tid, false);
 +    Utils.unreserveTable(tableId, tid, false);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index a643ccb,0000000..fbaa70d
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@@ -1,400 -1,0 +1,402 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.DataInput;
 +import java.io.DataOutput;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.commons.codec.binary.Hex;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.io.WritableUtils;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +class CompactionDriver extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private long compactId;
 +  private String tableId;
 +  private byte[] startRow;
 +  private byte[] endRow;
 +  private String namespaceId;
 +
 +  public CompactionDriver(long compactId, String tableId, byte[] startRow, byte[] endRow) {
 +
 +    this.compactId = compactId;
 +    this.tableId = tableId;
 +    this.startRow = startRow;
 +    this.endRow = endRow;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +
 +    String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    if (Long.parseLong(new String(zoo.getData(zCancelID, null))) >= compactId) {
 +      // compaction was canceled
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, "Compaction canceled");
 +    }
 +
 +    MapCounter<TServerInstance> serversToFlush = new MapCounter<TServerInstance>();
 +    Connector conn = master.getConnector();
 +
 +    Scanner scanner;
 +
 +    if (tableId.equals(MetadataTable.ID)) {
 +      scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
 +      scanner.setRange(MetadataSchema.TabletsSection.getRange());
 +    } else {
 +      scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 +      Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange();
 +      scanner.setRange(range);
 +    }
 +
 +    TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner);
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
 +    scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +
 +    long t1 = System.currentTimeMillis();
 +    RowIterator ri = new RowIterator(scanner);
 +
 +    int tabletsToWaitFor = 0;
 +    int tabletCount = 0;
 +
 +    while (ri.hasNext()) {
 +      Iterator<Entry<Key,Value>> row = ri.next();
 +      long tabletCompactID = -1;
 +
 +      TServerInstance server = null;
 +
 +      Entry<Key,Value> entry = null;
 +      while (row.hasNext()) {
 +        entry = row.next();
 +        Key key = entry.getKey();
 +
 +        if (TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +          tabletCompactID = Long.parseLong(entry.getValue().toString());
 +
 +        if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily()))
 +          server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
 +      }
 +
 +      if (tabletCompactID < compactId) {
 +        tabletsToWaitFor++;
 +        if (server != null)
 +          serversToFlush.increment(server, 1);
 +      }
 +
 +      tabletCount++;
 +
 +      Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
 +      if (tabletEndRow == null || (endRow != null && tabletEndRow.compareTo(new Text(endRow)) >= 0))
 +        break;
 +    }
 +
 +    long scanTime = System.currentTimeMillis() - t1;
 +
 +    Instance instance = master.getInstance();
 +    Tables.clearCache(instance);
 +    if (tabletCount == 0 && !Tables.exists(instance, tableId))
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
 +
 +    if (serversToFlush.size() == 0 && Tables.getTableState(instance, tableId) == TableState.OFFLINE)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
 +
 +    if (tabletsToWaitFor == 0)
 +      return 0;
 +
 +    for (TServerInstance tsi : serversToFlush.keySet()) {
 +      try {
 +        final TServerConnection server = master.getConnection(tsi);
 +        if (server != null)
 +          server.compact(master.getMasterLock(), tableId, startRow, endRow);
 +      } catch (TException ex) {
 +        Logger.getLogger(CompactionDriver.class).error(ex.toString());
 +      }
 +    }
 +
 +    long sleepTime = 500;
 +
 +    if (serversToFlush.size() > 0)
 +      sleepTime = Collections.max(serversToFlush.values()) * sleepTime; // make wait time depend on the server with the most to
 +                                                                        // compact
 +
 +    sleepTime = Math.max(2 * scanTime, sleepTime);
 +
 +    sleepTime = Math.min(sleepTime, 30000);
 +
 +    return sleepTime;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +    CompactRange.removeIterators(tid, tableId);
 +    Utils.getReadLock(tableId, tid).unlock();
 +    Utils.getReadLock(namespaceId, tid).unlock();
 +    return null;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +
 +  }
 +
 +}
 +
 +public class CompactRange extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +  private byte[] startRow;
 +  private byte[] endRow;
 +  private byte[] iterators;
 +  private String namespaceId;
 +
 +  public static class CompactionIterators implements Writable {
 +    byte[] startRow;
 +    byte[] endRow;
 +    List<IteratorSetting> iterators;
 +
 +    public CompactionIterators(byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) {
 +      this.startRow = startRow;
 +      this.endRow = endRow;
 +      this.iterators = iterators;
 +    }
 +
 +    public CompactionIterators() {
 +      startRow = null;
 +      endRow = null;
 +      iterators = Collections.emptyList();
 +    }
 +
 +    @Override
 +    public void write(DataOutput out) throws IOException {
 +      out.writeBoolean(startRow != null);
 +      if (startRow != null) {
 +        out.writeInt(startRow.length);
 +        out.write(startRow);
 +      }
 +
 +      out.writeBoolean(endRow != null);
 +      if (endRow != null) {
 +        out.writeInt(endRow.length);
 +        out.write(endRow);
 +      }
 +
 +      out.writeInt(iterators.size());
 +      for (IteratorSetting is : iterators) {
 +        is.write(out);
 +      }
 +    }
 +
 +    @Override
 +    public void readFields(DataInput in) throws IOException {
 +      if (in.readBoolean()) {
 +        startRow = new byte[in.readInt()];
 +        in.readFully(startRow);
 +      } else {
 +        startRow = null;
 +      }
 +
 +      if (in.readBoolean()) {
 +        endRow = new byte[in.readInt()];
 +        in.readFully(endRow);
 +      } else {
 +        endRow = null;
 +      }
 +
 +      int num = in.readInt();
 +      iterators = new ArrayList<IteratorSetting>(num);
 +
 +      for (int i = 0; i < num; i++) {
 +        iterators.add(new IteratorSetting(in));
 +      }
 +    }
 +
 +    public Text getEndRow() {
 +      if (endRow == null)
 +        return null;
 +      return new Text(endRow);
 +    }
 +
 +    public Text getStartRow() {
 +      if (startRow == null)
 +        return null;
 +      return new Text(startRow);
 +    }
 +
 +    public List<IteratorSetting> getIterators() {
 +      return iterators;
 +    }
 +  }
 +
 +  public CompactRange(String tableId, byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) throws ThriftTableOperationException {
 +    this.tableId = tableId;
 +    this.startRow = startRow.length == 0 ? null : startRow;
 +    this.endRow = endRow.length == 0 ? null : endRow;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +
 +    if (iterators.size() > 0) {
 +      this.iterators = WritableUtils.toByteArray(new CompactionIterators(this.startRow, this.endRow, iterators));
 +    } else {
 +      iterators = null;
 +    }
 +
 +    if (this.startRow != null && this.endRow != null && new Text(startRow).compareTo(new Text(endRow)) >= 0)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.BAD_RANGE,
 +          "start row must be less than end row");
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT)
 +        + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(final long tid, Master environment) throws Exception {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    byte[] cid;
 +    try {
 +      cid = zoo.mutate(zTablePath, null, null, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] currentValue) throws Exception {
-           String cvs = new String(currentValue, Constants.UTF8);
++          String cvs = new String(currentValue, UTF_8);
 +          String[] tokens = cvs.split(",");
 +          long flushID = Long.parseLong(tokens[0]);
 +          flushID++;
 +
 +          String txidString = String.format("%016x", tid);
 +
 +          for (int i = 1; i < tokens.length; i++) {
 +            if (tokens[i].startsWith(txidString))
 +              continue; // skip self
 +
 +            throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER,
 +                "Another compaction with iterators is running");
 +          }
 +
 +          StringBuilder encodedIterators = new StringBuilder();
 +
 +          if (iterators != null) {
 +            Hex hex = new Hex();
 +            encodedIterators.append(",");
 +            encodedIterators.append(txidString);
 +            encodedIterators.append("=");
-             encodedIterators.append(new String(hex.encode(iterators), Constants.UTF8));
++            encodedIterators.append(new String(hex.encode(iterators), UTF_8));
 +          }
 +
-           return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++          return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
 +        }
 +      });
 +
-       return new CompactionDriver(Long.parseLong(new String(cid, Constants.UTF8).split(",")[0]), tableId, startRow, endRow);
++      return new CompactionDriver(Long.parseLong(new String(cid, UTF_8).split(",")[0]), tableId, startRow, endRow);
 +    } catch (NoNodeException nne) {
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
 +    }
 +
 +  }
 +
 +  static void removeIterators(final long txid, String tableId) throws Exception {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    zoo.mutate(zTablePath, null, null, new Mutator() {
 +      @Override
 +      public byte[] mutate(byte[] currentValue) throws Exception {
-         String cvs = new String(currentValue, Constants.UTF8);
++        String cvs = new String(currentValue, UTF_8);
 +        String[] tokens = cvs.split(",");
 +        long flushID = Long.parseLong(tokens[0]);
 +
 +        String txidString = String.format("%016x", txid);
 +
 +        StringBuilder encodedIterators = new StringBuilder();
 +        for (int i = 1; i < tokens.length; i++) {
 +          if (tokens[i].startsWith(txidString))
 +            continue;
 +          encodedIterators.append(",");
 +          encodedIterators.append(tokens[i]);
 +        }
 +
-         return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++        return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
 +      }
 +    });
 +
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    try {
 +      removeIterators(tid, tableId);
 +    } finally {
 +      Utils.unreserveNamespace(namespaceId, tid, false);
 +      Utils.unreserveTable(tableId, tid, false);
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index eeb9b16,0000000..037455c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@@ -1,316 -1,0 +1,318 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedOutputStream;
 +import java.io.BufferedWriter;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.io.OutputStreamWriter;
 +import java.io.Serializable;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.zip.ZipEntry;
 +import java.util.zip.ZipOutputStream;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +
 +class ExportInfo implements Serializable {
 +  
 +  private static final long serialVersionUID = 1L;
 +  
 +  public String tableName;
 +  public String tableID;
 +  public String exportDir;
 +  public String namespaceID;
 +}
 +
 +class WriteExportFiles extends MasterRepo {
 +  
 +  private static final long serialVersionUID = 1L;
 +  private final ExportInfo tableInfo;
 +  
 +  WriteExportFiles(ExportInfo tableInfo) {
 +    this.tableInfo = tableInfo;
 +  }
 +  
 +  private void checkOffline(Connector conn) throws Exception {
 +    if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
 +      Tables.clearCache(conn.getInstance());
 +      if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
 +        throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
 +            "Table is not offline");
 +      }
 +    }
 +  }
 +  
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    
 +    long reserved = Utils.reserveNamespace(tableInfo.namespaceID, tid, false, true, TableOperation.EXPORT)
 +        + Utils.reserveTable(tableInfo.tableID, tid, false, true, TableOperation.EXPORT);
 +    if (reserved > 0)
 +      return reserved;
 +    
 +    Connector conn = master.getConnector();
 +    
 +    checkOffline(conn);
 +    
 +    Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    metaScanner.setRange(new KeyExtent(new Text(tableInfo.tableID), null, null).toMetadataRange());
 +    
 +    // scan for locations
 +    metaScanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +    metaScanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
 +    
 +    if (metaScanner.iterator().hasNext()) {
 +      return 500;
 +    }
 +    
 +    // use the same range to check for walogs that we used to check for hosted (or future hosted) tablets
 +    // this is done as a separate scan after we check for locations, because walogs are okay only if there is no location
 +    metaScanner.clearColumns();
 +    metaScanner.fetchColumnFamily(LogColumnFamily.NAME);
 +    
 +    if (metaScanner.iterator().hasNext()) {
 +      throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
 +          "Write ahead logs found for table");
 +    }
 +    
 +    return 0;
 +  }
 +  
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    Connector conn = master.getConnector();
 +    
 +    try {
 +      exportTable(master.getFileSystem(), conn, tableInfo.tableName, tableInfo.tableID, tableInfo.exportDir);
 +    } catch (IOException ioe) {
 +      throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
 +          "Failed to create export files " + ioe.getMessage());
 +    }
 +    Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
 +    Utils.unreserveTable(tableInfo.tableID, tid, false);
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +    return null;
 +  }
 +  
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
 +    Utils.unreserveTable(tableInfo.tableID, tid, false);
 +  }
 +  
 +  public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
 +    
 +    fs.mkdirs(new Path(exportDir));
 +    Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
 +    
 +    FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
 +    ZipOutputStream zipOut = new ZipOutputStream(fileOut);
 +    BufferedOutputStream bufOut = new BufferedOutputStream(zipOut);
 +    DataOutputStream dataOut = new DataOutputStream(bufOut);
 +    
 +    try {
 +      
 +      zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_INFO_FILE));
-       OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++      OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
 +      osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.VERSION + "\n");
 +      osw.append("srcInstanceName:" + conn.getInstance().getInstanceName() + "\n");
 +      osw.append("srcInstanceID:" + conn.getInstance().getInstanceID() + "\n");
 +      osw.append("srcZookeepers:" + conn.getInstance().getZooKeepers() + "\n");
 +      osw.append("srcTableName:" + tableName + "\n");
 +      osw.append("srcTableID:" + tableID + "\n");
 +      osw.append(ExportTable.DATA_VERSION_PROP + ":" + ServerConstants.DATA_VERSION + "\n");
 +      osw.append("srcCodeVersion:" + Constants.VERSION + "\n");
 +      
 +      osw.flush();
 +      dataOut.flush();
 +      
 +      exportConfig(conn, tableID, zipOut, dataOut);
 +      dataOut.flush();
 +      
 +      Map<String,String> uniqueFiles = exportMetadata(fs, conn, tableID, zipOut, dataOut);
 +      
 +      dataOut.close();
 +      dataOut = null;
 +      
 +      createDistcpFile(fs, exportDir, exportMetaFilePath, uniqueFiles);
 +      
 +    } finally {
 +      if (dataOut != null)
 +        dataOut.close();
 +    }
 +  }
 +  
 +  private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException {
-     BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), Constants.UTF8));
++    BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), UTF_8));
 +    
 +    try {
 +      for (String file : uniqueFiles.values()) {
 +        distcpOut.append(file);
 +        distcpOut.newLine();
 +      }
 +      
 +      distcpOut.append(exportMetaFilePath.toString());
 +      distcpOut.newLine();
 +      
 +      distcpOut.close();
 +      distcpOut = null;
 +      
 +    } finally {
 +      if (distcpOut != null)
 +        distcpOut.close();
 +    }
 +  }
 +  
 +  private static Map<String,String> exportMetadata(VolumeManager fs, Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut)
 +      throws IOException, TableNotFoundException {
 +    zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_METADATA_FILE));
 +    
 +    Map<String,String> uniqueFiles = new HashMap<String,String>();
 +    
 +    Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    metaScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(metaScanner);
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(metaScanner);
 +    metaScanner.setRange(new KeyExtent(new Text(tableID), null, null).toMetadataRange());
 +    
 +    for (Entry<Key,Value> entry : metaScanner) {
 +      entry.getKey().write(dataOut);
 +      entry.getValue().write(dataOut);
 +      
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        String path = fs.getFullPath(entry.getKey()).toString();
 +        String tokens[] = path.split("/");
 +        if (tokens.length < 1) {
 +          throw new RuntimeException("Illegal path " + path);
 +        }
 +        
 +        String filename = tokens[tokens.length - 1];
 +        
 +        String existingPath = uniqueFiles.get(filename);
 +        if (existingPath == null) {
 +          uniqueFiles.put(filename, path);
 +        } else if (!existingPath.equals(path)) {
 +          // make sure file names are unique, should only apply for tables with file names generated by Accumulo 1.3 and earlier
 +          throw new IOException("Cannot export table with nonunique file names " + filename + ". Major compact table.");
 +        }
 +        
 +      }
 +    }
 +    return uniqueFiles;
 +  }
 +  
 +  private static void exportConfig(Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws AccumuloException,
 +      AccumuloSecurityException, TableNotFoundException, IOException {
 +    
 +    DefaultConfiguration defaultConfig = AccumuloConfiguration.getDefaultConfiguration();
 +    Map<String,String> siteConfig = conn.instanceOperations().getSiteConfiguration();
 +    Map<String,String> systemConfig = conn.instanceOperations().getSystemConfiguration();
 +    
 +    TableConfiguration tableConfig = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableID);
 +    
-     OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++    OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
 +    
 +    // only put props that are different than defaults and higher level configurations
 +    zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));
 +    for (Entry<String,String> prop : tableConfig) {
 +      if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) {
 +        Property key = Property.getPropertyByKey(prop.getKey());
 +        
 +        if (key == null || !defaultConfig.get(key).equals(prop.getValue())) {
 +          if (!prop.getValue().equals(siteConfig.get(prop.getKey())) && !prop.getValue().equals(systemConfig.get(prop.getKey()))) {
 +            osw.append(prop.getKey() + "=" + prop.getValue() + "\n");
 +          }
 +        }
 +      }
 +    }
 +    
 +    osw.flush();
 +  }
 +}
 +
 +public class ExportTable extends MasterRepo {
 +  private static final long serialVersionUID = 1L;
 +  
 +  private final ExportInfo tableInfo;
 +  
 +  public ExportTable(String tableName, String tableId, String exportDir) {
 +    tableInfo = new ExportInfo();
 +    tableInfo.tableName = tableName;
 +    tableInfo.exportDir = exportDir;
 +    tableInfo.tableID = tableId;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    tableInfo.namespaceID = Tables.getNamespaceId(inst, tableId);
 +  }
 +  
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +  }
 +  
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    return new WriteExportFiles(tableInfo);
 +  }
 +  
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +  }
 +  
 +  public static final int VERSION = 1;
 +  
 +  public static final String DATA_VERSION_PROP = "srcDataVersion";
 +  public static final String EXPORT_VERSION_PROP = "exportVersion";
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 8de6bce,0000000..05ec263
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@@ -1,653 -1,0 +1,655 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedInputStream;
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.DataInputStream;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.io.OutputStreamWriter;
 +import java.io.Serializable;
 +import java.util.Arrays;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.zip.ZipEntry;
 +import java.util.zip.ZipInputStream;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.TableOperationsImpl;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.FastFormat;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.TablePropUtil;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +/**
 + *
 + */
 +class ImportedTableInfo implements Serializable {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  public String exportDir;
 +  public String user;
 +  public String tableName;
 +  public String tableId;
 +  public String importDir;
 +  public String namespaceId;
 +}
 +
 +class FinishImportTable extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  public FinishImportTable(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return 0;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +
 +    env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir, "mappings.txt"));
 +
 +    TableManager.getInstance().transitionTableState(tableInfo.tableId, TableState.ONLINE);
 +
 +    Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
 +    Utils.unreserveTable(tableInfo.tableId, tid, true);
 +
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +
 +    env.getEventCoordinator().event("Imported table %s ", tableInfo.tableName);
 +
 +    Logger.getLogger(FinishImportTable.class).debug("Imported table " + tableInfo.tableId + " " + tableInfo.tableName);
 +
 +    return null;
 +  }
 +
 +  @Override
 +  public String getReturn() {
 +    return tableInfo.tableId;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {}
 +
 +}
 +
 +class MoveExportedFiles extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  MoveExportedFiles(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    try {
 +      VolumeManager fs = master.getFileSystem();
 +
 +      Map<String,String> fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo);
 +
 +      for (String oldFileName : fileNameMappings.keySet()) {
 +        if (!fs.exists(new Path(tableInfo.exportDir, oldFileName))) {
 +          throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +              "File referenced by exported table does not exists " + oldFileName);
 +        }
 +      }
 +
 +      FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
 +
 +      for (FileStatus fileStatus : files) {
 +        String newName = fileNameMappings.get(fileStatus.getPath().getName());
 +
 +        if (newName != null)
 +          fs.rename(fileStatus.getPath(), new Path(tableInfo.importDir, newName));
 +      }
 +
 +      return new FinishImportTable(tableInfo);
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error renaming files " + ioe.getMessage());
 +    }
 +  }
 +}
 +
 +class PopulateMetadataTable extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  PopulateMetadataTable(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  static Map<String,String> readMappingFile(VolumeManager fs, ImportedTableInfo tableInfo) throws Exception {
-     BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), Constants.UTF8));
++    BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), UTF_8));
 +
 +    try {
 +      Map<String,String> map = new HashMap<String,String>();
 +
 +      String line = null;
 +      while ((line = in.readLine()) != null) {
 +        String sa[] = line.split(":", 2);
 +        map.put(sa[0], sa[1]);
 +      }
 +
 +      return map;
 +    } finally {
 +      in.close();
 +    }
 +
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +
 +    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
 +
 +    BatchWriter mbw = null;
 +    ZipInputStream zis = null;
 +
 +    try {
 +      VolumeManager fs = master.getFileSystem();
 +
 +      mbw = master.getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +
 +      zis = new ZipInputStream(fs.open(path));
 +
 +      Map<String,String> fileNameMappings = readMappingFile(fs, tableInfo);
 +
 +      log.info("importDir is " + tableInfo.importDir);
 +
 +      // This is a directory already prefixed with proper volume information e.g. hdfs://localhost:8020/path/to/accumulo/tables/...
 +      final String bulkDir = tableInfo.importDir;
 +
 +      final String[] tableDirs = ServerConstants.getTablesDirs();
 +
 +      ZipEntry zipEntry;
 +      while ((zipEntry = zis.getNextEntry()) != null) {
 +        if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) {
 +          DataInputStream in = new DataInputStream(new BufferedInputStream(zis));
 +
 +          Key key = new Key();
 +          Value val = new Value();
 +
 +          Mutation m = null;
 +          Text currentRow = null;
 +          int dirCount = 0;
 +
 +          while (true) {
 +            key.readFields(in);
 +            val.readFields(in);
 +
 +            Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow();
 +            Text metadataRow = new KeyExtent(new Text(tableInfo.tableId), endRow, null).getMetadataEntry();
 +
 +            Text cq;
 +
 +            if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +              String oldName = new Path(key.getColumnQualifier().toString()).getName();
 +              String newName = fileNameMappings.get(oldName);
 +
 +              if (newName == null) {
 +                throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +                    "File " + oldName + " does not exist in import dir");
 +              }
 +
 +              cq = new Text(bulkDir + "/" + newName);
 +            } else {
 +              cq = key.getColumnQualifier();
 +            }
 +
 +            if (m == null) {
 +              // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
-               String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++              String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
 +
 +              // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
 +              String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
 +
 +              m = new Mutation(metadataRow);
-               TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++              TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
 +              currentRow = metadataRow;
 +            }
 +
 +            if (!currentRow.equals(metadataRow)) {
 +              mbw.addMutation(m);
 +
 +              // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
-               String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++              String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
 +
 +              // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
 +              String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
 +
 +              m = new Mutation(metadataRow);
-               TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++              TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
 +            }
 +
 +            m.put(key.getColumnFamily(), cq, val);
 +
 +            if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
 +              mbw.addMutation(m);
 +              break; // its the last column in the last row
 +            }
 +          }
 +
 +          break;
 +        }
 +      }
 +
 +      return new MoveExportedFiles(tableInfo);
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error reading " + path + " " + ioe.getMessage());
 +    } finally {
 +      if (zis != null) {
 +        try {
 +          zis.close();
 +        } catch (IOException ioe) {
 +          log.warn("Failed to close zip file ", ioe);
 +        }
 +      }
 +
 +      if (mbw != null) {
 +        mbw.close();
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Given options for tables (across multiple volumes), construct an absolute path using the unique name within the chosen volume
 +   *
 +   * @return An absolute, unique path for the imported table
 +   */
 +  protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) {
 +    // We can try to spread out the tablet dirs across all volumes
 +    String tableDir = master.getFileSystem().choose(tableDirs);
 +
 +    // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
 +    return tableDir + "/" + tableInfo.tableId + "/" + tabletDir;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    MetadataTableUtil.deleteTable(tableInfo.tableId, false, SystemCredentials.get(), environment.getMasterLock());
 +  }
 +}
 +
 +class MapImportFileNames extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  MapImportFileNames(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +
 +    Path path = new Path(tableInfo.importDir, "mappings.txt");
 +
 +    BufferedWriter mappingsWriter = null;
 +
 +    try {
 +      VolumeManager fs = environment.getFileSystem();
 +
 +      fs.mkdirs(new Path(tableInfo.importDir));
 +
 +      FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
 +
 +      UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
-       mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), Constants.UTF8));
++      mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), UTF_8));
 +
 +      for (FileStatus fileStatus : files) {
 +        String fileName = fileStatus.getPath().getName();
 +        log.info("filename " + fileStatus.getPath().toString());
 +        String sa[] = fileName.split("\\.");
 +        String extension = "";
 +        if (sa.length > 1) {
 +          extension = sa[sa.length - 1];
 +
 +          if (!FileOperations.getValidExtensions().contains(extension)) {
 +            continue;
 +          }
 +        } else {
 +          // assume it is a map file
 +          extension = Constants.MAPFILE_EXTENSION;
 +        }
 +
 +        String newName = "I" + namer.getNextName() + "." + extension;
 +
 +        mappingsWriter.append(fileName);
 +        mappingsWriter.append(':');
 +        mappingsWriter.append(newName);
 +        mappingsWriter.newLine();
 +      }
 +
 +      mappingsWriter.close();
 +      mappingsWriter = null;
 +
 +      return new PopulateMetadataTable(tableInfo);
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error writing mapping file " + path + " " + ioe.getMessage());
 +    } finally {
 +      if (mappingsWriter != null)
 +        try {
 +          mappingsWriter.close();
 +        } catch (IOException ioe) {
 +          log.warn("Failed to close " + path, ioe);
 +        }
 +    }
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir));
 +  }
 +}
 +
 +class CreateImportDir extends MasterRepo {
 +  private static final Logger log = Logger.getLogger(CreateImportDir.class);
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  CreateImportDir(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +
 +    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
 +    Path exportDir = new Path(tableInfo.exportDir);
 +    String[] tableDirs = ServerConstants.getTablesDirs();
 +
 +    log.info("Looking for matching filesystem for " + exportDir + " from options " + Arrays.toString(tableDirs));
 +    Path base = master.getFileSystem().matchingFileSystem(exportDir, tableDirs);
 +    log.info("Chose base table directory of " + base);
 +    Path directory = new Path(base, tableInfo.tableId);
 +
 +    Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
 +
 +    tableInfo.importDir = newBulkDir.toString();
 +
 +    log.info("Using import dir: " + tableInfo.importDir);
 +
 +    return new MapImportFileNames(tableInfo);
 +  }
 +}
 +
 +class ImportPopulateZookeeper extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  ImportPopulateZookeeper(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT);
 +  }
 +
 +  private Map<String,String> getExportedProps(VolumeManager fs) throws Exception {
 +
 +    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
 +
 +    try {
 +      FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
 +      return TableOperationsImpl.getExportedProps(ns, path);
 +    } catch (IOException ioe) {
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error reading table props from " + path + " " + ioe.getMessage());
 +    }
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    // reserve the table name in zookeeper or fail
 +
 +    Utils.tableNameLock.lock();
 +    try {
 +      // write tableName & tableId to zookeeper
 +      Instance instance = HdfsZooInstance.getInstance();
 +
 +      Utils.checkTableDoesNotExist(instance, tableInfo.tableName, tableInfo.tableId, TableOperation.CREATE);
 +
 +      String namespace = Tables.qualify(tableInfo.tableName).getFirst();
 +      String namespaceId = Namespaces.getNamespaceId(instance, namespace);
 +      TableManager.getInstance().addTable(tableInfo.tableId, namespaceId, tableInfo.tableName, NodeExistsPolicy.OVERWRITE);
 +
 +      Tables.clearCache(instance);
 +    } finally {
 +      Utils.tableNameLock.unlock();
 +    }
 +
 +    for (Entry<String,String> entry : getExportedProps(env.getFileSystem()).entrySet())
 +      if (!TablePropUtil.setTableProperty(tableInfo.tableId, entry.getKey(), entry.getValue())) {
 +        throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +            "Invalid table property " + entry.getKey());
 +      }
 +
 +    return new CreateImportDir(tableInfo);
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Instance instance = HdfsZooInstance.getInstance();
 +    TableManager.getInstance().removeTable(tableInfo.tableId);
 +    Utils.unreserveTable(tableInfo.tableId, tid, true);
 +    Tables.clearCache(instance);
 +  }
 +}
 +
 +class ImportSetupPermissions extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  public ImportSetupPermissions(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return 0;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    // give all table permissions to the creator
 +    SecurityOperation security = AuditedSecurityOperation.getInstance();
 +    for (TablePermission permission : TablePermission.values()) {
 +      try {
 +        security.grantTablePermission(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.user, tableInfo.tableId, permission, tableInfo.namespaceId);
 +      } catch (ThriftSecurityException e) {
 +        Logger.getLogger(ImportSetupPermissions.class).error(e.getMessage(), e);
 +        throw e;
 +      }
 +    }
 +
 +    // setup permissions in zookeeper before table info in zookeeper
 +    // this way concurrent users will not get a spurious permission denied
 +    // error
 +    return new ImportPopulateZookeeper(tableInfo);
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    AuditedSecurityOperation.getInstance().deleteTable(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.tableId, tableInfo.namespaceId);
 +  }
 +}
 +
 +public class ImportTable extends MasterRepo {
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  public ImportTable(String user, String tableName, String exportDir, String namespaceId) {
 +    tableInfo = new ImportedTableInfo();
 +    tableInfo.tableName = tableName;
 +    tableInfo.user = user;
 +    tableInfo.exportDir = exportDir;
 +    tableInfo.namespaceId = namespaceId;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid)
 +        + Utils.reserveNamespace(tableInfo.namespaceId, tid, false, true, TableOperation.IMPORT);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    checkVersions(env);
 +
 +    // first step is to reserve a table id.. if the machine fails during this step
 +    // it is ok to retry... the only side effect is that a table id may not be used
 +    // or skipped
 +
 +    // assuming only the master process is creating tables
 +
 +    Utils.idLock.lock();
 +    try {
 +      Instance instance = HdfsZooInstance.getInstance();
 +      tableInfo.tableId = Utils.getNextTableId(tableInfo.tableName, instance);
 +      return new ImportSetupPermissions(tableInfo);
 +    } finally {
 +      Utils.idLock.unlock();
 +    }
 +  }
 +
 +  public void checkVersions(Master env) throws ThriftTableOperationException {
 +    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
 +
 +    ZipInputStream zis = null;
 +
 +    try {
 +      zis = new ZipInputStream(env.getFileSystem().open(path));
 +
 +      Integer exportVersion = null;
 +      Integer dataVersion = null;
 +
 +      ZipEntry zipEntry;
 +      while ((zipEntry = zis.getNextEntry()) != null) {
 +        if (zipEntry.getName().equals(Constants.EXPORT_INFO_FILE)) {
-           BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++          BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
 +          String line = null;
 +          while ((line = in.readLine()) != null) {
 +            String sa[] = line.split(":", 2);
 +            if (sa[0].equals(ExportTable.EXPORT_VERSION_PROP)) {
 +              exportVersion = Integer.parseInt(sa[1]);
 +            } else if (sa[0].equals(ExportTable.DATA_VERSION_PROP)) {
 +              dataVersion = Integer.parseInt(sa[1]);
 +            }
 +          }
 +
 +          break;
 +        }
 +      }
 +
 +      zis.close();
 +      zis = null;
 +
 +      if (exportVersion == null || exportVersion > ExportTable.VERSION)
 +        throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +            "Incompatible export version " + exportVersion);
 +
 +      if (dataVersion == null || dataVersion > ServerConstants.DATA_VERSION)
 +        throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +            "Incompatible data version " + exportVersion);
 +
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Failed to read export metadata " + ioe.getMessage());
 +    } finally {
 +      if (zis != null)
 +        try {
 +          zis.close();
 +        } catch (IOException ioe) {
 +          log.warn(ioe.getMessage(), ioe);
 +        }
 +    }
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +    Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
 +  }
 +}


[11/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index 292129d,0000000..7ae4d5a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@@ -1,150 -1,0 +1,152 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.watcher;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Appender;
 +import org.apache.log4j.LogManager;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.helpers.FileWatchdog;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * Watcher that updates the monitor's log4j port from ZooKeeper in a system property
 + */
 +public class MonitorLog4jWatcher extends FileWatchdog implements Watcher {
 +  private static final Logger log = Logger.getLogger(MonitorLog4jWatcher.class);
 +
 +  private static final String HOST_PROPERTY_NAME = "org.apache.accumulo.core.host.log";
 +  private static final String PORT_PROPERTY_NAME = "org.apache.accumulo.core.host.log.port";
 +
 +  private final Object lock;
 +  private final Log4jConfiguration logConfig;
 +  private boolean loggingDisabled = false;
 +  protected String path;
 +
 +  public MonitorLog4jWatcher(String instance, String filename) {
 +    super(filename);
 +    this.path = ZooUtil.getRoot(instance) + Constants.ZMONITOR_LOG4J_ADDR;
 +    this.lock = new Object();
 +    this.logConfig = new Log4jConfiguration(filename);
 +    doOnChange();
 +  }
 +
 +  boolean isUsingProperties() {
 +    return logConfig.isUsingProperties();
 +  }
 +
 +  String getPath() {
 +    return path;
 +  }
 +
 +  @Override
 +  public void run() {
 +    try {
 +      // Initially set the logger if the Monitor's log4j advertisement node exists
 +      if (ZooReaderWriter.getInstance().exists(path, this))
 +        updateMonitorLog4jLocation();
 +      log.info("Set watch for Monitor Log4j watcher");
 +    } catch (Exception e) {
 +      log.error("Unable to set watch for Monitor Log4j watcher on " + path);
 +    }
 +
 +    super.run();
 +  }
 +
 +  @Override
 +  public void doOnChange() {
 +    // this method gets called in the parent class' constructor
 +    // I'm not sure of a better way to get around this. The final modifier helps though.
 +    if (null == lock) {
 +      return;
 +    }
 +
 +    synchronized (lock) {
 +      // We might triggered by file-reloading or from ZK update.
 +      // Either way will result in log-forwarding being restarted
 +      loggingDisabled = false;
 +      log.info("Enabled log-forwarding");
 +      logConfig.resetLogger();
 +    }
 +  }
 +
 +  @Override
 +  public void process(WatchedEvent event) {
 +    // We got an update, process the data in the node
 +    updateMonitorLog4jLocation();
 +
 +    if (event.getPath() != null) {
 +      try {
 +        ZooReaderWriter.getInstance().exists(event.getPath(), this);
 +      } catch (Exception ex) {
 +        log.error("Unable to reset watch for Monitor Log4j watcher", ex);
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Read the host and port information for the Monitor's log4j socket and update the system properties so that, on logger refresh, it sees the new information.
 +   */
 +  protected void updateMonitorLog4jLocation() {
 +    try {
-       String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), Constants.UTF8);
++      String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), UTF_8);
 +      HostAndPort hostAndPort = HostAndPort.fromString(hostPortString);
 +
 +      System.setProperty(HOST_PROPERTY_NAME, hostAndPort.getHostText());
 +      System.setProperty(PORT_PROPERTY_NAME, Integer.toString(hostAndPort.getPort()));
 +
 +      log.info("Changing monitor log4j address to " + hostAndPort.toString());
 +
 +      doOnChange();
 +    } catch (NoNodeException e) {
 +      // Not sure on the synchronization guarantees for Loggers and Appenders
 +      // on configuration reload
 +      synchronized (lock) {
 +        // Don't need to try to re-disable'ing it.
 +        if (loggingDisabled) {
 +          return;
 +        }
 +
 +        Logger logger = LogManager.getLogger("org.apache.accumulo");
 +        if (null != logger) {
 +          // TODO ACCUMULO-2343 Create a specific appender for log-forwarding to the monitor
 +          // that can replace the AsyncAppender+SocketAppender.
 +          Appender appender = logger.getAppender("ASYNC");
 +          if (null != appender) {
 +            log.info("Closing log-forwarding appender");
 +            appender.close();
 +            log.info("Removing log-forwarding appender");
 +            logger.removeAppender(appender);
 +            loggingDisabled = true;
 +          }
 +        }
 +      }
 +    } catch (IllegalArgumentException e) {
 +      log.error("Could not parse host and port information", e);
 +    } catch (Exception e) {
 +      log.error("Error reading zookeeper data for Monitor Log4j watcher", e);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
index b02e5d4,0000000..1bbae03
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
@@@ -1,54 -1,0 +1,55 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.zookeeper;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.locks.Lock;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class ZooQueueLock extends org.apache.accumulo.fate.zookeeper.ZooQueueLock {
 +
 +  public ZooQueueLock(String path, boolean ephemeral) throws KeeperException, InterruptedException {
 +    super(ZooReaderWriter.getInstance(), path, ephemeral);
 +  }
 +
 +  public static void main(String args[]) throws InterruptedException, KeeperException {
 +    ZooQueueLock lock = new ZooQueueLock("/lock", true);
-     DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(Constants.UTF8));
-     DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(Constants.UTF8));
++    DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(UTF_8));
++    DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(UTF_8));
 +    final Lock readLock = rlocker.readLock();
 +    readLock.lock();
 +    final Lock readLock2 = rlocker.readLock();
 +    readLock2.lock();
 +    final Lock writeLock = wlocker.writeLock();
 +    if (writeLock.tryLock(100, TimeUnit.MILLISECONDS))
 +      throw new RuntimeException("Write lock achieved during read lock!");
 +    readLock.unlock();
 +    readLock2.unlock();
 +    writeLock.lock();
 +    if (readLock.tryLock(100, TimeUnit.MILLISECONDS))
 +      throw new RuntimeException("Read lock achieved during write lock!");
-     final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(Constants.UTF8));
++    final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(UTF_8));
 +    writeLock2.unlock();
 +    readLock.lock();
 +    System.out.println("success");
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
index 435591d,0000000..c5374a2
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
@@@ -1,42 -1,0 +1,43 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.zookeeper;
 +
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +
 +public class ZooReaderWriter extends org.apache.accumulo.fate.zookeeper.ZooReaderWriter {
 +  private static final String SCHEME = "digest";
 +  private static final String USER = "accumulo";
 +  private static ZooReaderWriter instance = null;
 +  
 +  public ZooReaderWriter(String string, int timeInMillis, String secret) {
-     super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++    super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
 +  }
 +  
 +  public static synchronized ZooReaderWriter getInstance() {
 +    if (instance == null) {
 +      AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
 +      instance = new ZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
 +          conf.get(Property.INSTANCE_SECRET));
 +    }
 +    return instance;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index 4a7111d,0000000..e15e63e
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@@ -1,64 -1,0 +1,65 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.zookeeper;
 +
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +
 +/**
 + * A factory for {@link ZooReaderWriter} objects.
 + */
 +public class ZooReaderWriterFactory {
 +  private static final String SCHEME = "digest";
 +  private static final String USER = "accumulo";
 +  private static IZooReaderWriter instance = null;
 +
 +  /**
 +   * Gets a new reader/writer.
 +   *
 +   * @param string
 +   *          ZooKeeper connection string
 +   * @param timeInMillis
 +   *          session timeout in milliseconds
 +   * @param secret
 +   *          instance secret
 +   * @return reader/writer
 +   */
 +  public IZooReaderWriter getZooReaderWriter(String string, int timeInMillis, String secret) {
-     return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++    return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
 +  }
 +
 +  /**
 +   * Gets a reader/writer, retrieving ZooKeeper information from the site configuration. The same instance may be returned for multiple calls.
 +   *
 +   * @return reader/writer
 +   */
 +  public IZooReaderWriter getInstance() {
 +    synchronized (ZooReaderWriterFactory.class) {
 +      if (instance == null) {
 +        AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
 +        instance = getZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
 +            conf.get(Property.INSTANCE_SECRET));
 +      }
 +      return instance;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index e7f42af,0000000..3aed7ef
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@@ -1,159 -1,0 +1,161 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collection;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.UUID;
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.Before;
 +import org.junit.Test;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.eq;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.easymock.EasyMock.verify;
 +
 +public class NamespaceConfigurationTest {
 +  private static final String NSID = "namespace";
 +  private static final String ZOOKEEPERS = "localhost";
 +  private static final int ZK_SESSION_TIMEOUT = 120000;
 +
 +  private String iid;
 +  private Instance instance;
 +  private AccumuloConfiguration parent;
 +  private ZooCacheFactory zcf;
 +  private ZooCache zc;
 +  private NamespaceConfiguration c;
 +
 +  @Before
 +  public void setUp() {
 +    iid = UUID.randomUUID().toString();
 +    instance = createMock(Instance.class);
 +    parent = createMock(AccumuloConfiguration.class);
 +    c = new NamespaceConfiguration(NSID, instance, parent);
 +    zcf = createMock(ZooCacheFactory.class);
 +    c.setZooCacheFactory(zcf);
 +
 +    expect(instance.getInstanceID()).andReturn(iid);
 +    expectLastCall().anyTimes();
 +    expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
 +    expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
 +    replay(instance);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
 +    replay(zcf);
 +  }
 +
 +  @Test
 +  public void testGetters() {
 +    assertEquals(NSID, c.getNamespaceId());
 +    assertEquals(parent, c.getParentConfiguration());
 +  }
 +
 +  @Test
 +  public void testGet_InZK() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
-         "sekrit".getBytes(Constants.UTF8));
++        "sekrit".getBytes(UTF_8));
 +    replay(zc);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGet_InParent() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(null);
 +    replay(zc);
 +    expect(parent.get(p)).andReturn("sekrit");
 +    replay(parent);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGet_SkipParentIfAccumuloNS() {
 +    c = new NamespaceConfiguration(Namespaces.ACCUMULO_NAMESPACE_ID, instance, parent);
 +    c.setZooCacheFactory(zcf);
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + Namespaces.ACCUMULO_NAMESPACE_ID + Constants.ZNAMESPACE_CONF + "/" + p.getKey()))
 +        .andReturn(null);
 +    replay(zc);
 +    assertNull(c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGetProperties() {
 +    PropertyFilter filter = new AllFilter();
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add("foo");
 +    children.add("ding");
 +    expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF)).andReturn(children);
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "foo")).andReturn(
-         "bar".getBytes(Constants.UTF8));
++        "bar".getBytes(UTF_8));
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "ding")).andReturn(
-         "dong".getBytes(Constants.UTF8));
++        "dong".getBytes(UTF_8));
 +    replay(zc);
 +    c.getProperties(props, filter);
 +    assertEquals(2, props.size());
 +    assertEquals("bar", props.get("foo"));
 +    assertEquals("dong", props.get("ding"));
 +  }
 +
 +  @Test
 +  public void testObserver() {
 +    ConfigurationObserver o = createMock(ConfigurationObserver.class);
 +    c.addObserver(o);
 +    Collection<ConfigurationObserver> os = c.getObservers();
 +    assertEquals(1, os.size());
 +    assertTrue(os.contains(o));
 +    c.removeObserver(o);
 +    os = c.getObservers();
 +    assertEquals(0, os.size());
 +  }
 +
 +  @Test
 +  public void testInvalidateCache() {
 +    // need to do a get so the accessor is created
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
-         "sekrit".getBytes(Constants.UTF8));
++        "sekrit".getBytes(UTF_8));
 +    zc.clear();
 +    replay(zc);
 +    c.get(Property.INSTANCE_SECRET);
 +    c.invalidateCache();
 +    verify(zc);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
index 7f07d94,0000000..81148a0
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
@@@ -1,134 -1,0 +1,134 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.endsWith;
 +import static org.easymock.EasyMock.eq;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertSame;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.After;
 +import org.junit.Before;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +public class ServerConfigurationFactoryTest {
 +  private static final String ZK_HOST = "localhost";
 +  private static final int ZK_TIMEOUT = 120000;
 +  private static final String IID = "iid";
 +
 +  // use the same mock ZooCacheFactory and ZooCache for all tests
 +  private static ZooCacheFactory zcf;
 +  private static ZooCache zc;
 +
 +  @BeforeClass
 +  public static void setUpClass() throws Exception {
 +    zcf = createMock(ZooCacheFactory.class);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache(eq(ZK_HOST), eq(ZK_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
 +    expectLastCall().anyTimes();
 +    expect(zcf.getZooCache(ZK_HOST, ZK_TIMEOUT)).andReturn(zc);
 +    expectLastCall().anyTimes();
 +    replay(zcf);
 +
 +    expect(zc.getChildren(anyObject(String.class))).andReturn(null);
 +    expectLastCall().anyTimes();
 +    // ConfigSanityCheck looks at timeout
-     expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(Constants.UTF8));
++    expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(UTF_8));
 +    replay(zc);
 +  }
 +
 +  private Instance instance;
 +  private ServerConfigurationFactory scf;
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    instance = createMock(Instance.class);
 +    expect(instance.getInstanceID()).andReturn(IID);
 +    expectLastCall().anyTimes();
 +  }
 +
 +  @After
 +  public void tearDown() throws Exception {
 +    ServerConfigurationFactory.clearCachedConfigurations();
 +  }
 +
 +  private void mockInstanceForConfig() {
 +    expect(instance.getZooKeepers()).andReturn(ZK_HOST);
 +    expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_TIMEOUT);
 +  }
 +
 +  private void ready() {
 +    replay(instance);
 +    scf = new ServerConfigurationFactory(instance);
 +    scf.setZooCacheFactory(zcf);
 +  }
 +
 +  @Test
 +  public void testGetInstance() {
 +    ready();
 +    assertSame(instance, scf.getInstance());
 +  }
 +
 +  @Test
 +  public void testGetDefaultConfiguration() {
 +    ready();
 +    DefaultConfiguration c = scf.getDefaultConfiguration();
 +    assertNotNull(c);
 +  }
 +
 +  @Test
 +  public void testGetSiteConfiguration() {
 +    ready();
 +    SiteConfiguration c = scf.getSiteConfiguration();
 +    assertNotNull(c);
 +  }
 +
 +  @Test
 +  public void testGetConfiguration() {
 +    mockInstanceForConfig();
 +    ready();
 +    AccumuloConfiguration c = scf.getConfiguration();
 +    assertNotNull(c);
 +  }
 +
 +  private static final String NSID = "NAMESPACE";
 +
 +  @Test
 +  public void testGetNamespaceConfiguration() {
 +    mockInstanceForConfig();
 +    ready();
 +    NamespaceConfiguration c = scf.getNamespaceConfiguration(NSID);
 +    assertEquals(NSID, c.getNamespaceId());
 +
 +    assertSame(c, scf.getNamespaceConfiguration(NSID));
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
index 92580f3,0000000..2f38e4c
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
@@@ -1,145 -1,0 +1,146 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.eq;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.easymock.EasyMock.verify;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.Collection;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class TableConfigurationTest {
 +  private static final String TID = "table";
 +  private static final String ZOOKEEPERS = "localhost";
 +  private static final int ZK_SESSION_TIMEOUT = 120000;
 +
 +  private String iid;
 +  private Instance instance;
 +  private NamespaceConfiguration parent;
 +  private ZooCacheFactory zcf;
 +  private ZooCache zc;
 +  private TableConfiguration c;
 +
 +  @Before
 +  public void setUp() {
 +    iid = UUID.randomUUID().toString();
 +    instance = createMock(Instance.class);
 +    parent = createMock(NamespaceConfiguration.class);
 +    c = new TableConfiguration(iid, instance, TID, parent);
 +    zcf = createMock(ZooCacheFactory.class);
 +    c.setZooCacheFactory(zcf);
 +
 +    expect(instance.getInstanceID()).andReturn(iid);
 +    expectLastCall().anyTimes();
 +    expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
 +    expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
 +    replay(instance);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(TableConfWatcher.class))).andReturn(zc);
 +    replay(zcf);
 +  }
 +
 +  @Test
 +  public void testGetters() {
 +    assertEquals(TID, c.getTableId());
 +    assertEquals(parent, c.getParentConfiguration());
 +  }
 +
 +  @Test
 +  public void testGet_InZK() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
-         .andReturn("sekrit".getBytes(Constants.UTF8));
++        .andReturn("sekrit".getBytes(UTF_8));
 +    replay(zc);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGet_InParent() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(null);
 +    replay(zc);
 +    expect(parent.get(p)).andReturn("sekrit");
 +    replay(parent);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGetProperties() {
 +    PropertyFilter filter = new AllFilter();
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add("foo");
 +    children.add("ding");
 +    expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF)).andReturn(children);
-     expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(Constants.UTF8));
-     expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(Constants.UTF8));
++    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(UTF_8));
++    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(UTF_8));
 +    replay(zc);
 +    c.getProperties(props, filter);
 +    assertEquals(2, props.size());
 +    assertEquals("bar", props.get("foo"));
 +    assertEquals("dong", props.get("ding"));
 +  }
 +
 +  @Test
 +  public void testObserver() {
 +    ConfigurationObserver o = createMock(ConfigurationObserver.class);
 +    c.addObserver(o);
 +    Collection<ConfigurationObserver> os = c.getObservers();
 +    assertEquals(1, os.size());
 +    assertTrue(os.contains(o));
 +    c.removeObserver(o);
 +    os = c.getObservers();
 +    assertEquals(0, os.size());
 +  }
 +
 +  @Test
 +  public void testInvalidateCache() {
 +    // need to do a get so the accessor is created
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
-         .andReturn("sekrit".getBytes(Constants.UTF8));
++        .andReturn("sekrit".getBytes(UTF_8));
 +    zc.clear();
 +    replay(zc);
 +    c.get(Property.INSTANCE_SECRET);
 +    c.invalidateCache();
 +    verify(zc);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
index 1b9d8f7,0000000..4369119
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
@@@ -1,186 -1,0 +1,186 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.replay;
 +import static org.easymock.EasyMock.verify;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertSame;
 +
 +import java.util.List;
 +import java.util.Map;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ZooCachePropertyAccessorTest {
 +  private static final String PATH = "/root/path/to/props";
 +  private static final Property PROP = Property.INSTANCE_SECRET;
 +  private static final String KEY = PROP.getKey();
 +  private static final String FULL_PATH = PATH + "/" + KEY;
 +  private static final String VALUE = "value";
-   private static final byte[] VALUE_BYTES = VALUE.getBytes(Constants.UTF8);
++  private static final byte[] VALUE_BYTES = VALUE.getBytes(UTF_8);
 +
 +  private ZooCache zc;
 +  private ZooCachePropertyAccessor a;
 +
 +  @Before
 +  public void setUp() {
 +    zc = createMock(ZooCache.class);
 +    a = new ZooCachePropertyAccessor(zc);
 +  }
 +
 +  @Test
 +  public void testGetter() {
 +    assertSame(zc, a.getZooCache());
 +  }
 +
 +  @Test
 +  public void testGet_Valid() {
 +    expect(zc.get(FULL_PATH)).andReturn(VALUE_BYTES);
 +    replay(zc);
 +    assertEquals(VALUE, a.get(PROP, PATH, null));
 +  }
 +
 +  @Test
 +  public void testGet_Parent() {
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    expect(parent.get(PROP)).andReturn(VALUE);
 +    replay(parent);
 +    expect(zc.get(FULL_PATH)).andReturn(null);
 +    replay(zc);
 +    assertEquals(VALUE, a.get(PROP, PATH, parent));
 +  }
 +
 +  @Test
 +  public void testGet_Parent_Null() {
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    expect(parent.get(PROP)).andReturn(null);
 +    replay(parent);
 +    expect(zc.get(FULL_PATH)).andReturn(null);
 +    replay(zc);
 +    assertNull(a.get(PROP, PATH, parent));
 +  }
 +
 +  @Test
 +  public void testGet_Null_NoParent() {
 +    expect(zc.get(FULL_PATH)).andReturn(null);
 +    replay(zc);
 +    assertNull(a.get(PROP, PATH, null));
 +  }
 +
 +  @Test
 +  public void testGet_InvalidFormat() {
 +    Property badProp = Property.MASTER_CLIENTPORT;
 +    expect(zc.get(PATH + "/" + badProp.getKey())).andReturn(VALUE_BYTES);
 +    replay(zc);
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    expect(parent.get(badProp)).andReturn("12345");
 +    replay(parent);
 +    assertEquals("12345", a.get(badProp, PATH, parent));
 +  }
 +
 +  @Test
 +  public void testGetProperties() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    String child1 = "child1";
 +    String child2 = "child2";
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add(child1);
 +    children.add(child2);
 +    expect(zc.getChildren(PATH)).andReturn(children);
 +    expect(zc.get(PATH + "/" + child1)).andReturn(VALUE_BYTES);
 +    expect(zc.get(PATH + "/" + child2)).andReturn(null);
 +    replay(zc);
 +    expect(filter.accept(child1)).andReturn(true);
 +    expect(filter.accept(child2)).andReturn(true);
 +    replay(filter);
 +
 +    a.getProperties(props, PATH, filter, parent, null);
 +    assertEquals(1, props.size());
 +    assertEquals(VALUE, props.get(child1));
 +    verify(parent);
 +  }
 +
 +  @Test
 +  public void testGetProperties_NoChildren() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    expect(zc.getChildren(PATH)).andReturn(null);
 +    replay(zc);
 +
 +    a.getProperties(props, PATH, filter, parent, null);
 +    assertEquals(0, props.size());
 +  }
 +
 +  @Test
 +  public void testGetProperties_Filter() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    String child1 = "child1";
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add(child1);
 +    expect(zc.getChildren(PATH)).andReturn(children);
 +    replay(zc);
 +    expect(filter.accept(child1)).andReturn(false);
 +    replay(filter);
 +
 +    a.getProperties(props, PATH, filter, parent, null);
 +    assertEquals(0, props.size());
 +  }
 +
 +  @Test
 +  public void testGetProperties_ParentFilter() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    PropertyFilter parentFilter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, parentFilter);
 +    replay(parent);
 +    expect(zc.getChildren(PATH)).andReturn(null);
 +    replay(zc);
 +
 +    a.getProperties(props, PATH, filter, parent, parentFilter);
 +    verify(parent);
 +  }
 +
 +  @Test
 +  public void testInvalidateCache() {
 +    zc.clear();
 +    replay(zc);
 +    a.invalidateCache();
 +    verify(zc);
 +  }
 +}


[38/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
index cd26743,0000000..8022c14
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
@@@ -1,210 -1,0 +1,211 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.nio.ByteBuffer;
- import java.nio.charset.Charset;
 +import java.util.List;
 +
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +import static org.easymock.EasyMock.*;
 +import static org.junit.Assert.*;
 +
 +public class ValueTest {
 +  private static final byte[] toBytes(String s) {
-     return s.getBytes(Charset.forName("UTF-8"));
++    return s.getBytes(UTF_8);
 +  }
 +
 +  private static final byte[] DATA = toBytes("data");
 +  private static final ByteBuffer DATABUFF = ByteBuffer.allocate(DATA.length);
 +  static {
 +    DATABUFF.put(DATA);
 +  }
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    DATABUFF.rewind();
 +  }
 +
 +  @Test
 +  public void testDefault() {
 +    Value v = new Value();
 +    assertEquals(0, v.get().length);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullBytesConstructor() {
 +    new Value((byte[]) null);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullCopyConstructor() {
 +    new Value((Value) null);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullByteBufferConstructor() {
 +    new Value((ByteBuffer)null);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullSet() {
 +    Value v = new Value();
 +    v.set(null);
 +  }
 +
 +  @Test
 +  public void testByteArray() {
 +    Value v = new Value(DATA);
 +    assertArrayEquals(DATA, v.get());
 +    assertSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteArrayCopy() {
 +    Value v = new Value(DATA, true);
 +    assertArrayEquals(DATA, v.get());
 +    assertNotSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteBuffer() {
 +    Value v = new Value(DATABUFF);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteBufferCopy() {
 +    @SuppressWarnings("deprecation")
 +    Value v = new Value(DATABUFF, true);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testValueCopy() {
 +    Value ov = createMock(Value.class);
 +    expect(ov.get()).andReturn(DATA);
 +    expect(ov.getSize()).andReturn(4);
 +    replay(ov);
 +    Value v = new Value(ov);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteArrayOffsetLength() {
 +    Value v = new Value(DATA, 0, 4);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testSet() {
 +    Value v = new Value();
 +    v.set(DATA);
 +    assertArrayEquals(DATA, v.get());
 +    assertSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testCopy() {
 +    Value v = new Value();
 +    v.copy(DATA);
 +    assertArrayEquals(DATA, v.get());
 +    assertNotSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testGetSize() {
 +    Value v = new Value(DATA);
 +    assertEquals(DATA.length, v.getSize());
 +  }
 +
 +  @Test
 +  public void testGetSizeDefault() {
 +    Value v = new Value();
 +    assertEquals(0, v.getSize());
 +  }
 +
 +  @Test
 +  public void testWriteRead() throws Exception {
 +    Value v = new Value(DATA);
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    DataOutputStream dos = new DataOutputStream(baos);
 +    v.write(dos);
 +    dos.close();
 +    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
 +    DataInputStream dis = new DataInputStream(bais);
 +    Value v2 = new Value();
 +    v2.readFields(dis);
 +    dis.close();
 +    assertArrayEquals(DATA, v2.get());
 +  }
 +
 +  @Test
 +  public void testHashCode() {
 +    Value v1 = new Value(DATA);
 +    Value v2 = new Value(DATA);
 +    assertEquals(v1.hashCode(), v2.hashCode());
 +  }
 +
 +  @Test
 +  public void testCompareTo() {
 +    Value v1 = new Value(DATA);
 +    Value v2 = new Value(toBytes("datb"));
 +    assertTrue(v1.compareTo(v2) < 0);
 +    assertTrue(v2.compareTo(v1) > 0);
 +    Value v1a = new Value(DATA);
 +    assertTrue(v1.compareTo(v1a) == 0);
 +    Value v3 = new Value(toBytes("datc"));
 +    assertTrue(v2.compareTo(v3) < 0);
 +    assertTrue(v1.compareTo(v3) < 0);
 +  }
 +
 +  @Test
 +  public void testEquals() {
 +    Value v1 = new Value(DATA);
 +    assertTrue(v1.equals(v1));
 +    Value v2 = new Value(DATA);
 +    assertTrue(v1.equals(v2));
 +    assertTrue(v2.equals(v1));
 +    Value v3 = new Value(toBytes("datb"));
 +    assertFalse(v1.equals(v3));
 +  }
 +
 +  @Test
 +  public void testToArray() {
 +    List<byte[]> l = new java.util.ArrayList<byte[]>();
 +    byte[] one = toBytes("one");
 +    byte[] two = toBytes("two");
 +    byte[] three = toBytes("three");
 +    l.add(one);
 +    l.add(two);
 +    l.add(three);
 +
 +    byte[][] a = Value.toArray(l);
 +    assertEquals(3, a.length);
 +    assertArrayEquals(one, a[0]);
 +    assertArrayEquals(two, a[1]);
 +    assertArrayEquals(three, a[2]);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index 931ff41,6c4e814..2577e48
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@@ -16,19 -16,11 +16,19 @@@
   */
  package org.apache.accumulo.core.security;
  
++import static com.google.common.base.Charsets.UTF_8;
  import static org.apache.accumulo.core.security.ColumnVisibility.quote;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
  import static org.junit.Assert.fail;
  
 +import java.util.Comparator;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.security.ColumnVisibility.Node;
 +import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator;
 +import org.apache.accumulo.core.security.ColumnVisibility.NodeType;
 +import org.apache.hadoop.io.Text;
  import org.junit.Test;
  
  public class ColumnVisibilityTest {
@@@ -166,83 -152,4 +166,83 @@@
      cv = new ColumnVisibility(quote("五"));
      assertEquals("[\"五\"]", cv.toString());
    }
 +
 +  @Test
 +  public void testParseTree() {
 +    Node node = parse("(W)|(U&V)");
 +    assertNode(node, NodeType.OR, 0, 9);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
 +    assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithNoChildren() {
 +    Node node = parse("ABC");
 +    assertNode(node, NodeType.TERM, 0, 3);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithTwoChildren() {
 +    Node node = parse("ABC|DEF");
 +    assertNode(node, NodeType.OR, 0, 7);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
 +    assertNode(node.getChildren().get(1), NodeType.TERM, 4, 7);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithParenthesesAndTwoChildren() {
 +    Node node = parse("(ABC|DEF)");
 +    assertNode(node, NodeType.OR, 1, 8);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 1, 4);
 +    assertNode(node.getChildren().get(1), NodeType.TERM, 5, 8);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithParenthesizedChildren() {
 +    Node node = parse("ABC|(DEF&GHI)");
 +    assertNode(node, NodeType.OR, 0, 13);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
 +    assertNode(node.getChildren().get(1), NodeType.AND, 5, 12);
 +    assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 8);
 +    assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 9, 12);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithMoreParentheses() {
 +    Node node = parse("(W)|(U&V)");
 +    assertNode(node, NodeType.OR, 0, 9);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
 +    assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
 +    assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 6);
 +    assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 7, 8);
 +  }
 +
 +  @Test
 +  public void testEmptyParseTreesAreEqual() {
 +    Comparator<Node> comparator = new NodeComparator(new byte[] {});
 +    Node empty = new ColumnVisibility().getParseTree();
 +    assertEquals(0, comparator.compare(empty, parse("")));
 +  }
 +
 +  @Test
 +  public void testParseTreesOrdering() {
-     byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(Constants.UTF8);
++    byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(UTF_8);
 +    byte[] flattened = new ColumnVisibility(expression).flatten();
 +
 +    // Convert to String for indexOf convenience
-     String flat = new String(flattened, Constants.UTF8);
++    String flat = new String(flattened, UTF_8);
 +    assertTrue("shortest expressions sort first", flat.indexOf('e') < flat.indexOf('|'));
 +    assertTrue("shortest children sort first", flat.indexOf('b') < flat.indexOf('a'));
 +  }
 +
 +  private Node parse(String s) {
 +    ColumnVisibility v = new ColumnVisibility(s);
 +    return v.getParseTree();
 +  }
 +
 +  private void assertNode(Node node, NodeType nodeType, int start, int end) {
 +    assertEquals(node.type, nodeType);
 +    assertEquals(start, node.start);
 +    assertEquals(end, node.end);
 +  }
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
index b925e4a,0000000..a4e9f02
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
@@@ -1,143 -1,0 +1,143 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +import static org.junit.Assert.fail;
 +
 +import javax.security.auth.DestroyFailedException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.client.security.tokens.NullToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.junit.Test;
 +
 +/**
 + * 
 + */
 +public class CredentialsTest {
 +  
 +  @Test
 +  public void testToThrift() throws DestroyFailedException {
 +    // verify thrift serialization
 +    Credentials creds = new Credentials("test", new PasswordToken("testing"));
 +    TCredentials tCreds = creds.toThrift(new MockInstance());
 +    assertEquals("test", tCreds.getPrincipal());
 +    assertEquals(PasswordToken.class.getName(), tCreds.getTokenClassName());
 +    assertArrayEquals(AuthenticationTokenSerializer.serialize(new PasswordToken("testing")), tCreds.getToken());
 +    
 +    // verify that we can't serialize if it's destroyed
 +    creds.getToken().destroy();
 +    try {
 +      creds.toThrift(new MockInstance());
 +      fail();
 +    } catch (Exception e) {
 +      assertTrue(e instanceof RuntimeException);
 +      assertTrue(e.getCause() instanceof AccumuloSecurityException);
 +      assertTrue(AccumuloSecurityException.class.cast(e.getCause()).getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
 +    }
 +  }
 +
 +  @Test
 +  public void roundtripThrift() throws DestroyFailedException {
 +    Credentials creds = new Credentials("test", new PasswordToken("testing"));
 +    TCredentials tCreds = creds.toThrift(new MockInstance());
 +    Credentials roundtrip = Credentials.fromThrift(tCreds);
 +    assertEquals("Roundtrip through thirft changed credentials equality", creds, roundtrip);
 +  }
 +  
 +  @Test
 +  public void testMockConnector() throws AccumuloException, DestroyFailedException, AccumuloSecurityException {
 +    Instance inst = new MockInstance();
 +    Connector rootConnector = inst.getConnector("root", new PasswordToken());
 +    PasswordToken testToken = new PasswordToken("testPass");
 +    rootConnector.securityOperations().createLocalUser("testUser", testToken);
 +    
 +    assertFalse(testToken.isDestroyed());
 +    testToken.destroy();
 +    assertTrue(testToken.isDestroyed());
 +    try {
 +      inst.getConnector("testUser", testToken);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      assertTrue(e.getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
 +    }
 +  }
 +  
 +  @Test
 +  public void testEqualsAndHashCode() {
 +    Credentials nullNullCreds = new Credentials(null, null);
 +    Credentials abcNullCreds = new Credentials("abc", new NullToken());
 +    Credentials cbaNullCreds = new Credentials("cba", new NullToken());
 +    Credentials abcBlahCreds = new Credentials("abc", new PasswordToken("blah"));
 +    
 +    // check hash codes
 +    assertEquals(0, nullNullCreds.hashCode());
 +    assertEquals("abc".hashCode(), abcNullCreds.hashCode());
 +    assertEquals(abcNullCreds.hashCode(), abcBlahCreds.hashCode());
 +    assertFalse(abcNullCreds.hashCode() == cbaNullCreds.hashCode());
 +    
 +    // identity
 +    assertEquals(abcNullCreds, abcNullCreds);
 +    assertEquals(new Credentials("abc", new NullToken()), abcNullCreds);
 +    // equal, but different token constructors
-     assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(Constants.UTF8))), new Credentials("abc", new PasswordToken("abc")));
++    assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(UTF_8))), new Credentials("abc", new PasswordToken("abc")));
 +    // test not equals
 +    assertFalse(nullNullCreds.equals(abcBlahCreds));
 +    assertFalse(nullNullCreds.equals(abcNullCreds));
 +    assertFalse(abcNullCreds.equals(abcBlahCreds));
 +  }
 +  
 +  @Test
 +  public void testCredentialsSerialization() throws AccumuloSecurityException {
-     Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(Constants.UTF8)));
++    Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(UTF_8)));
 +    String serialized = creds.serialize();
 +    Credentials result = Credentials.deserialize(serialized);
 +    assertEquals(creds, result);
 +    assertEquals("a:b-c", result.getPrincipal());
 +    assertEquals(new PasswordToken("d-e-f"), result.getToken());
 +    
 +    Credentials nullNullCreds = new Credentials(null, null);
 +    serialized = nullNullCreds.serialize();
 +    result = Credentials.deserialize(serialized);
 +    assertEquals(null, result.getPrincipal());
 +    assertEquals(null, result.getToken());
 +  }
 +  
 +  @Test
 +  public void testToString() {
 +    Credentials creds = new Credentials(null, null);
 +    assertEquals(Credentials.class.getName() + ":null:null:<hidden>", creds.toString());
 +    creds = new Credentials("", new NullToken());
 +    assertEquals(Credentials.class.getName() + "::" + NullToken.class.getName() + ":<hidden>", creds.toString());
 +    creds = new Credentials("abc", null);
 +    assertEquals(Credentials.class.getName() + ":abc:null:<hidden>", creds.toString());
 +    creds = new Credentials("abc", new PasswordToken(""));
 +    assertEquals(Credentials.class.getName() + ":abc:" + PasswordToken.class.getName() + ":<hidden>", creds.toString());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index d31a788,0000000..523279a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -1,103 -1,0 +1,103 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.createNiceMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +
 +import java.util.Arrays;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.constraints.Constraint.Environment;
 +import org.apache.accumulo.core.data.ArrayByteSequence;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.junit.Before;
 +import org.junit.Ignore;
 +import org.junit.Test;
 +
 +public class VisibilityConstraintTest {
 +
 +  VisibilityConstraint vc;
 +  Environment env;
 +  Mutation mutation;
 +
 +  static final ColumnVisibility good = new ColumnVisibility("good");
 +  static final ColumnVisibility bad = new ColumnVisibility("bad");
 +
 +  static final String D = "don't care";
 +
 +  static final List<Short> ENOAUTH = Arrays.asList((short) 2);
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    vc = new VisibilityConstraint();
 +    mutation = new Mutation("r");
 +
-     ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
++    ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(UTF_8));
 +
 +    AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
 +    expect(ac.contains(bs)).andReturn(true);
 +    replay(ac);
 +
 +    env = createMock(Environment.class);
 +    expect(env.getAuthorizationsContainer()).andReturn(ac);
 +    replay(env);
 +  }
 +
 +  @Test
 +  public void testNoVisibility() {
 +    mutation.put(D, D, D);
 +    assertNull("authorized", vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testVisibilityNoAuth() {
 +    mutation.put(D, D, bad, D);
 +    assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testGoodVisibilityAuth() {
 +    mutation.put(D, D, good, D);
 +    assertNull("authorized", vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testCachedVisibilities() {
 +    mutation.put(D, D, good, "v");
 +    mutation.put(D, D, good, "v2");
 +    assertNull("authorized", vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testMixedVisibilities() {
 +    mutation.put(D, D, bad, D);
 +    mutation.put(D, D, good, D);
 +    assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  @Ignore
 +  public void testMalformedVisibility() {
 +    // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index ea64ac7,0000000..9078ef9
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -1,162 -1,0 +1,162 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security.crypto;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.IOException;
 +import java.util.Arrays;
 +import java.util.Random;
 +
- import org.apache.accumulo.core.Constants;
 +import org.junit.Test;
 +
 +public class BlockedIOStreamTest {
 +  @Test
 +  public void testLargeBlockIO() throws IOException {
 +    writeRead(1024, 2048);
 +  }
 +
 +  private void writeRead(int blockSize, int expectedSize) throws IOException {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, 1);
 +
 +    String contentString = "My Blocked Content String";
-     byte[] content = contentString.getBytes(Constants.UTF8);
++    byte[] content = contentString.getBytes(UTF_8);
 +    blockOut.write(content);
 +    blockOut.flush();
 +
 +    String contentString2 = "My Other Blocked Content String";
-     byte[] content2 = contentString2.getBytes(Constants.UTF8);
++    byte[] content2 = contentString2.getBytes(UTF_8);
 +    blockOut.write(content2);
 +    blockOut.flush();
 +
 +    blockOut.close();
 +    byte[] written = baos.toByteArray();
 +    assertEquals(expectedSize, written.length);
 +
 +    ByteArrayInputStream biis = new ByteArrayInputStream(written);
 +    BlockedInputStream blockIn = new BlockedInputStream(biis, blockSize, blockSize);
 +    DataInputStream dIn = new DataInputStream(blockIn);
 +
 +    dIn.readFully(content, 0, content.length);
-     String readContentString = new String(content, Constants.UTF8);
++    String readContentString = new String(content, UTF_8);
 +
 +    assertEquals(contentString, readContentString);
 +
 +    dIn.readFully(content2, 0, content2.length);
-     String readContentString2 = new String(content2, Constants.UTF8);
++    String readContentString2 = new String(content2, UTF_8);
 +
 +    assertEquals(contentString2, readContentString2);
 +
 +    blockIn.close();
 +  }
 +
 +  @Test
 +  public void testSmallBufferBlockedIO() throws IOException {
 +    writeRead(16, (12 + 4) * (int) (Math.ceil(25.0 / 12) + Math.ceil(31.0 / 12)));
 +  }
 +
 +  @Test
 +  public void testSpillingOverOutputStream() throws IOException {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    // buffer will be size 12
 +    BlockedOutputStream blockOut = new BlockedOutputStream(baos, 16, 16);
 +    Random r = new Random(22);
 +
 +    byte[] undersized = new byte[11];
 +    byte[] perfectSized = new byte[12];
 +    byte[] overSized = new byte[13];
 +    byte[] perfectlyOversized = new byte[13];
 +    byte filler = (byte) r.nextInt();
 +
 +    r.nextBytes(undersized);
 +    r.nextBytes(perfectSized);
 +    r.nextBytes(overSized);
 +    r.nextBytes(perfectlyOversized);
 +
 +    // 1 block
 +    blockOut.write(undersized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    // 2 blocks
 +    blockOut.write(perfectSized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    // 2 blocks
 +    blockOut.write(overSized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    // 3 blocks
 +    blockOut.write(undersized);
 +    blockOut.write(perfectlyOversized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    blockOut.close();
 +    assertEquals(16 * 8, baos.toByteArray().length);
 +  }
 +
 +  @Test
 +  public void testGiantWrite() throws IOException {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    int blockSize = 16;
 +    // buffer will be size 12
 +    BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, blockSize);
 +    Random r = new Random(22);
 +
 +    int size = 1024 * 1024 * 128;
 +    byte[] giant = new byte[size];
 +    byte[] pattern = new byte[1024];
 +
 +    r.nextBytes(pattern);
 +
 +    for (int i = 0; i < size / 1024; i++) {
 +      System.arraycopy(pattern, 0, giant, i * 1024, 1024);
 +    }
 +
 +    blockOut.write(giant);
 +    blockOut.flush();
 +
 +    blockOut.close();
 +    baos.close();
 +
 +    int blocks = (int) Math.ceil(size / (blockSize - 4.0));
 +    byte[] byteStream = baos.toByteArray();
 +
 +    assertEquals(blocks * 16, byteStream.length);
 +
 +    DataInputStream blockIn = new DataInputStream(new BlockedInputStream(new ByteArrayInputStream(byteStream), blockSize, blockSize));
 +    Arrays.fill(giant, (byte) 0);
 +    blockIn.readFully(giant, 0, size);
 +    blockIn.close();
 +
 +    for (int i = 0; i < size / 1024; i++) {
 +      byte[] readChunk = new byte[1024];
 +      System.arraycopy(giant, i * 1024, readChunk, 0, 1024);
 +      assertArrayEquals(pattern, readChunk);
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
index 36def05,0000000..a2e4f7a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
@@@ -1,176 -1,0 +1,176 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util.format;
 +
- import static org.apache.accumulo.core.Constants.UTF8;
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.createNiceMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.util.Collections;
 +import java.util.Map;
 +import java.util.TreeMap;
 +
 +import jline.UnsupportedTerminal;
 +import jline.console.ConsoleReader;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.util.shell.Shell;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class DeleterFormatterTest {
 +  DeleterFormatter formatter;
 +  Map<Key,Value> data;
 +  BatchWriter writer;
 +  BatchWriter exceptionWriter;
 +  Shell shellState;
 +
 +  ByteArrayOutputStream baos;
 +  ConsoleReader reader;
 +
 +  SettableInputStream input;
 +
 +  class SettableInputStream extends InputStream {
 +    ByteArrayInputStream bais;
 +
 +    @Override
 +    public int read() throws IOException {
 +      return bais.read();
 +    }
 +
 +    public void set(String in) {
-       bais = new ByteArrayInputStream(in.getBytes(UTF8));
++      bais = new ByteArrayInputStream(in.getBytes(UTF_8));
 +    }
 +  };
 +
 +  @Before
 +  public void setUp() throws IOException, MutationsRejectedException {
 +    input = new SettableInputStream();
 +    baos = new ByteArrayOutputStream();
 +
 +    MutationsRejectedException mre = createMock(MutationsRejectedException.class);
 +
 +    writer = createNiceMock(BatchWriter.class);
 +    exceptionWriter = createNiceMock(BatchWriter.class);
 +    exceptionWriter.close();
 +    expectLastCall().andThrow(mre);
 +    exceptionWriter.addMutation(anyObject(Mutation.class));
 +    expectLastCall().andThrow(mre);
 +
 +    shellState = createNiceMock(Shell.class);
 +
 +    reader = new ConsoleReader(input, baos, new UnsupportedTerminal());
 +    expect(shellState.getReader()).andReturn(reader).anyTimes();
 +
 +    replay(writer, exceptionWriter, shellState);
 +
 +    data = new TreeMap<Key,Value>();
-     data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF8)));
++    data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF_8)));
 +  }
 +
 +  @Test
 +  public void testEmpty() {
 +    formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true);
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testSingle() throws IOException {
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[DELETED]", " r ", "cf", "cq", "value");
 +  }
 +
 +  @Test
 +  public void testNo() throws IOException {
 +    input.set("no\n");
-     data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[SKIPPED]", " r ", "cf", "cq", "value");
 +
 +    assertTrue(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testNoConfirmation() throws IOException {
 +    input.set("");
-     data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[SKIPPED]", " r ", "cf", "cq", "value");
 +
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testYes() throws IOException {
 +    input.set("y\nyes\n");
-     data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    verify("[DELETED]", " r ", "cf", "cq", "value");
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    verify("[DELETED]", " z ", "v2");
 +  }
 +
 +  @Test
 +  public void testMutationException() {
 +    formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  private void verify(String... chunks) throws IOException {
 +    reader.flush();
 +
 +    String output = baos.toString();
 +    for (String chunk : chunks) {
 +      assertTrue(output.contains(chunk));
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 8fc68fd,0000000..eef06c3
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@@ -1,90 -1,0 +1,90 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util.format;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.Arrays;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.TreeMap;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.commons.collections.CollectionUtils;
 +import org.apache.commons.collections.Predicate;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ShardedTableDistributionFormatterTest {
 +  ShardedTableDistributionFormatter formatter;
 +
 +  Map<Key,Value> data;
 +
 +  @Before
 +  public void setUp() {
 +    data = new TreeMap<Key,Value>();
 +    formatter = new ShardedTableDistributionFormatter();
 +  }
 +
 +  @Test
 +  public void testInitialize() {
 +    data.put(new Key(), new Value());
 +    data.put(new Key("r", "~tab"), new Value());
 +    formatter.initialize(data.entrySet(), false);
 +
 +    assertTrue(formatter.hasNext());
 +    formatter.next();
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testAggregate() {
-     data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(Constants.UTF8)));
-     data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(Constants.UTF8)));
-     data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(Constants.UTF8)));
++    data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(UTF_8)));
++    data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(UTF_8)));
++    data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(UTF_8)));
 +
 +    formatter.initialize(data.entrySet(), false);
 +
 +    String[] resultLines = formatter.next().split("\n");
 +    List<String> results = Arrays.asList(resultLines).subList(2, 4);
 +
 +    assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("NULL", 1)));
 +    assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("19700101", 2)));
 +
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  private static class AggregateReportChecker implements Predicate {
 +    private String day;
 +    private int count;
 +
 +    AggregateReportChecker(String day, int count) {
 +      this.day = day;
 +      this.count = count;
 +    }
 +
 +    @Override
 +    public boolean evaluate(Object arg) {
 +      String resLine = (String) arg;
 +      return resLine.startsWith(this.day) && resLine.endsWith("" + this.count);
 +    }
 +
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 1f8c32d,e411d22..af31cf2
--- a/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@@ -16,11 -16,10 +16,12 @@@
   */
  package org.apache.accumulo.fate;
  
- import java.nio.charset.Charset;
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
 +import java.util.EnumSet;
 +import java.util.Formatter;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
@@@ -37,34 -35,8 +38,33 @@@ import org.apache.zookeeper.KeeperExcep
   * A utility to administer FATE operations
   */
  public class AdminUtil<T> {
-   private static final Charset UTF8 = Charset.forName("UTF-8");
    
 +  private boolean exitOnError = false;
 +  
 +  /**
 +   * Default constructor
 +   */
 +  public AdminUtil() {
 +    this(true);
 +  }
 +  
 +  /**
 +   * Constructor
 +   * 
 +   * @param exitOnError
 +   *          <code>System.exit(1)</code> on error if true
 +   */
 +  public AdminUtil(boolean exitOnError) {
 +    super();
 +    this.exitOnError = exitOnError;
 +  }
 +  
    public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath) throws KeeperException, InterruptedException {
 +    print(zs, zk, lockPath, new Formatter(System.out), null, null);
 +  }
 +  
 +  public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath, Formatter fmt, Set<Long> filterTxid, EnumSet<TStatus> filterStatus)
 +      throws KeeperException, InterruptedException {
      Map<Long,List<String>> heldLocks = new HashMap<Long,List<String>>();
      Map<Long,List<String>> waitingLocks = new HashMap<Long,List<String>>();
      

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index b6890c6,3ef6b0a..ff35364
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -32,8 -33,7 +33,7 @@@ import org.apache.zookeeper.ZooKeeper
  import org.apache.zookeeper.ZooKeeper.States;
  
  public class ZooSession {
-   private static final Charset UTF8 = Charset.forName("UTF-8");
 -  
 +
    public static class ZooSessionShutdownException extends RuntimeException {
  
      private static final long serialVersionUID = 1L;
@@@ -46,19 -46,18 +46,19 @@@
      public ZooSessionInfo(ZooKeeper zooKeeper, ZooWatcher watcher) {
        this.zooKeeper = zooKeeper;
      }
 -    
 +
      ZooKeeper zooKeeper;
    }
 -  
 +
    private static Map<String,ZooSessionInfo> sessions = new HashMap<String,ZooSessionInfo>();
 -  
 +
    private static String sessionKey(String keepers, int timeout, String scheme, byte[] auth) {
-     return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF8));
+     return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF_8));
    }
 -  
 +
    private static class ZooWatcher implements Watcher {
 -    
 +
 +    @Override
      public void process(WatchedEvent event) {
        if (event.getState() == KeeperState.Expired) {
          log.debug("Session expired, state of current session : " + event.getState());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index d5dc1e9,0000000..8216441
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -1,851 -1,0 +1,853 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.minicluster.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.File;
 +import java.io.FileFilter;
 +import java.io.FileWriter;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.InputStreamReader;
 +import java.net.InetSocketAddress;
 +import java.net.Socket;
 +import java.net.URI;
 +import java.net.URISyntaxException;
 +import java.net.URL;
 +import java.net.URLClassLoader;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Properties;
 +import java.util.Set;
 +import java.util.concurrent.Callable;
 +import java.util.concurrent.ExecutionException;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.FutureTask;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.TimeoutException;
 +
 +import org.apache.accumulo.cluster.AccumuloCluster;
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.impl.MasterClient;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.gc.SimpleGarbageCollector;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.master.state.SetGoalState;
 +import org.apache.accumulo.minicluster.ServerType;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.init.Initialize;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.AccumuloStatus;
 +import org.apache.accumulo.server.util.PortUtils;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
 +import org.apache.accumulo.start.Main;
 +import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.accumulo.tserver.TabletServer;
 +import org.apache.commons.configuration.MapConfiguration;
 +import org.apache.commons.io.FileUtils;
 +import org.apache.commons.vfs2.FileObject;
 +import org.apache.commons.vfs2.impl.VFSClassLoader;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.CommonConfigurationKeys;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.hdfs.DFSConfigKeys;
 +import org.apache.hadoop.hdfs.MiniDFSCluster;
 +import org.apache.thrift.TException;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.data.Stat;
 +import org.apache.zookeeper.server.ZooKeeperServerMain;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.base.Predicate;
 +import com.google.common.collect.Maps;
 +
 +/**
 + * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
 + * code against a real Accumulo instance. Its much more accurate for testing than {@link org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower.
 + *
 + * @since 1.6.0
 + */
 +public class MiniAccumuloClusterImpl implements AccumuloCluster {
 +  private static final Logger log = LoggerFactory.getLogger(MiniAccumuloClusterImpl.class);
 +
 +  public static class LogWriter extends Daemon {
 +    private BufferedReader in;
 +    private BufferedWriter out;
 +
 +    public LogWriter(InputStream stream, File logFile) throws IOException {
 +      this.in = new BufferedReader(new InputStreamReader(stream));
 +      out = new BufferedWriter(new FileWriter(logFile));
 +
 +      SimpleTimer.getInstance().schedule(new Runnable() {
 +        @Override
 +        public void run() {
 +          try {
 +            flush();
 +          } catch (IOException e) {
 +            e.printStackTrace();
 +          }
 +        }
 +      }, 1000, 1000);
 +    }
 +
 +    public synchronized void flush() throws IOException {
 +      if (out != null)
 +        out.flush();
 +    }
 +
 +    @Override
 +    public void run() {
 +      String line;
 +
 +      try {
 +        while ((line = in.readLine()) != null) {
 +          out.append(line);
 +          out.append("\n");
 +        }
 +
 +        synchronized (this) {
 +          out.close();
 +          out = null;
 +          in.close();
 +        }
 +
 +      } catch (IOException e) {}
 +    }
 +  }
 +
 +  private boolean initialized = false;
 +  private Process zooKeeperProcess = null;
 +  private Process masterProcess = null;
 +  private Process gcProcess = null;
 +  private List<Process> tabletServerProcesses = Collections.synchronizedList(new ArrayList<Process>());
 +
 +  private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
 +
 +  private File zooCfgFile;
 +  private String dfsUri;
 +
 +  public List<LogWriter> getLogWriters() {
 +    return logWriters;
 +  }
 +
 +  private List<LogWriter> logWriters = new ArrayList<MiniAccumuloClusterImpl.LogWriter>();
 +
 +  private MiniAccumuloConfigImpl config;
 +  private MiniDFSCluster miniDFS = null;
 +  private List<Process> cleanup = new ArrayList<Process>();
 +
 +  private ExecutorService executor;
 +
 +  public Process exec(Class<?> clazz, String... args) throws IOException {
 +    return exec(clazz, null, args);
 +  }
 +
 +  public Process exec(Class<?> clazz, List<String> jvmArgs, String... args) throws IOException {
 +    ArrayList<String> jvmArgs2 = new ArrayList<String>(1 + (jvmArgs == null ? 0 : jvmArgs.size()));
 +    jvmArgs2.add("-Xmx" + config.getDefaultMemory());
 +    if (jvmArgs != null)
 +      jvmArgs2.addAll(jvmArgs);
 +    Process proc = _exec(clazz, jvmArgs2, args);
 +    cleanup.add(proc);
 +    return proc;
 +  }
 +
 +  private boolean containsSiteFile(File f) {
 +    return f.isDirectory() && f.listFiles(new FileFilter() {
 +
 +      @Override
 +      public boolean accept(File pathname) {
 +        return pathname.getName().endsWith("site.xml");
 +      }
 +    }).length > 0;
 +  }
 +
 +  private void append(StringBuilder classpathBuilder, URL url) throws URISyntaxException {
 +    File file = new File(url.toURI());
 +    // do not include dirs containing hadoop or accumulo site files
 +    if (!containsSiteFile(file))
 +      classpathBuilder.append(File.pathSeparator).append(file.getAbsolutePath());
 +  }
 +
 +  private String getClasspath() throws IOException {
 +
 +    try {
 +      ArrayList<ClassLoader> classloaders = new ArrayList<ClassLoader>();
 +
 +      ClassLoader cl = this.getClass().getClassLoader();
 +
 +      while (cl != null) {
 +        classloaders.add(cl);
 +        cl = cl.getParent();
 +      }
 +
 +      Collections.reverse(classloaders);
 +
 +      StringBuilder classpathBuilder = new StringBuilder();
 +      classpathBuilder.append(config.getConfDir().getAbsolutePath());
 +
 +      if (config.getHadoopConfDir() != null)
 +        classpathBuilder.append(File.pathSeparator).append(config.getHadoopConfDir().getAbsolutePath());
 +
 +      if (config.getClasspathItems() == null) {
 +
 +        // assume 0 is the system classloader and skip it
 +        for (int i = 1; i < classloaders.size(); i++) {
 +          ClassLoader classLoader = classloaders.get(i);
 +
 +          if (classLoader instanceof URLClassLoader) {
 +
 +            URLClassLoader ucl = (URLClassLoader) classLoader;
 +
 +            for (URL u : ucl.getURLs()) {
 +              append(classpathBuilder, u);
 +            }
 +
 +          } else if (classLoader instanceof VFSClassLoader) {
 +
 +            VFSClassLoader vcl = (VFSClassLoader) classLoader;
 +            for (FileObject f : vcl.getFileObjects()) {
 +              append(classpathBuilder, f.getURL());
 +            }
 +          } else {
 +            throw new IllegalArgumentException("Unknown classloader type : " + classLoader.getClass().getName());
 +          }
 +        }
 +      } else {
 +        for (String s : config.getClasspathItems())
 +          classpathBuilder.append(File.pathSeparator).append(s);
 +      }
 +
 +      return classpathBuilder.toString();
 +
 +    } catch (URISyntaxException e) {
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  private Process _exec(Class<?> clazz, List<String> extraJvmOpts, String... args) throws IOException {
 +    String javaHome = System.getProperty("java.home");
 +    String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
 +    String classpath = getClasspath();
 +
 +    String className = clazz.getName();
 +
 +    ArrayList<String> argList = new ArrayList<String>();
 +    argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath));
 +    argList.addAll(extraJvmOpts);
 +    for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) {
 +      argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue()));
 +    }
 +    argList.addAll(Arrays.asList("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-Dapple.awt.UIElement=true", Main.class.getName(),
 +        className));
 +    argList.addAll(Arrays.asList(args));
 +
 +    ProcessBuilder builder = new ProcessBuilder(argList);
 +
 +    builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
 +    builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
 +    builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath());
 +    String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator);
 +    builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath);
 +    builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath);
 +
 +    // if we're running under accumulo.start, we forward these env vars
 +    String env = System.getenv("HADOOP_PREFIX");
 +    if (env != null)
 +      builder.environment().put("HADOOP_PREFIX", env);
 +    env = System.getenv("ZOOKEEPER_HOME");
 +    if (env != null)
 +      builder.environment().put("ZOOKEEPER_HOME", env);
 +    builder.environment().put("ACCUMULO_CONF_DIR", config.getConfDir().getAbsolutePath());
 +    // hadoop-2.2 puts error messages in the logs if this is not set
 +    builder.environment().put("HADOOP_HOME", config.getDir().getAbsolutePath());
 +    if (config.getHadoopConfDir() != null)
 +      builder.environment().put("HADOOP_CONF_DIR", config.getHadoopConfDir().getAbsolutePath());
 +
 +    Process process = builder.start();
 +
 +    LogWriter lw;
 +    lw = new LogWriter(process.getErrorStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
 +    logWriters.add(lw);
 +    lw.start();
 +    lw = new LogWriter(process.getInputStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
 +    logWriters.add(lw);
 +    lw.start();
 +
 +    return process;
 +  }
 +
 +  private Process _exec(Class<?> clazz, ServerType serverType, String... args) throws IOException {
 +
 +    List<String> jvmOpts = new ArrayList<String>();
 +    jvmOpts.add("-Xmx" + config.getMemory(serverType));
 +
 +    if (config.isJDWPEnabled()) {
 +      Integer port = PortUtils.getRandomFreePort();
 +      jvmOpts.addAll(buildRemoteDebugParams(port));
 +      debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
 +    }
 +    return _exec(clazz, jvmOpts, args);
 +  }
 +
 +  /**
 +   *
 +   * @param dir
 +   *          An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
 +   *          and Junit provide methods for creating temporary directories.
 +   * @param rootPassword
 +   *          Initial root password for instance.
 +   */
 +  public MiniAccumuloClusterImpl(File dir, String rootPassword) throws IOException {
 +    this(new MiniAccumuloConfigImpl(dir, rootPassword));
 +  }
 +
 +  /**
 +   * @param config
 +   *          initial configuration
 +   */
 +  @SuppressWarnings("deprecation")
 +  public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException {
 +
 +    this.config = config.initialize();
 +
 +    config.getConfDir().mkdirs();
 +    config.getLogDir().mkdirs();
 +    config.getLibDir().mkdirs();
 +    config.getLibExtDir().mkdirs();
 +
 +    if (!config.useExistingInstance()) {
 +      config.getZooKeeperDir().mkdirs();
 +      config.getWalogDir().mkdirs();
 +      config.getAccumuloDir().mkdirs();
 +    }
 +
 +    if (config.useMiniDFS()) {
 +      File nn = new File(config.getAccumuloDir(), "nn");
 +      nn.mkdirs();
 +      File dn = new File(config.getAccumuloDir(), "dn");
 +      dn.mkdirs();
 +      File dfs = new File(config.getAccumuloDir(), "dfs");
 +      dfs.mkdirs();
 +      Configuration conf = new Configuration();
 +      conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
 +      conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
 +      conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
 +      conf.set("dfs.support.append", "true");
 +      conf.set("dfs.datanode.synconclose", "true");
 +      conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
 +      String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath());
 +      miniDFS = new MiniDFSCluster(conf, 1, true, null);
 +      if (oldTestBuildData == null)
 +        System.clearProperty("test.build.data");
 +      else
 +        System.setProperty("test.build.data", oldTestBuildData);
 +      miniDFS.waitClusterUp();
 +      InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
 +      dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
 +      File coreFile = new File(config.getConfDir(), "core-site.xml");
 +      writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
 +      File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
 +      writeConfig(hdfsFile, conf);
 +
 +      Map<String,String> siteConfig = config.getSiteConfig();
 +      siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
 +      siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
 +      config.setSiteConfig(siteConfig);
 +    } else if (config.useExistingInstance()) {
 +      dfsUri = CachedConfiguration.getInstance().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
 +    } else {
 +      dfsUri = "file://";
 +    }
 +
 +    File clientConfFile = config.getClientConfFile();
 +    // Write only the properties that correspond to ClientConfiguration properties
 +    writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate<Entry<String,String>>() {
 +      @Override
 +      public boolean apply(Entry<String,String> v) {
 +        return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null;
 +      }
 +    }));
 +
 +    File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
 +    writeConfig(siteFile, config.getSiteConfig().entrySet());
 +
 +    if (!config.useExistingInstance()) {
 +      zooCfgFile = new File(config.getConfDir(), "zoo.cfg");
 +      FileWriter fileWriter = new FileWriter(zooCfgFile);
 +
 +      // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
 +      Properties zooCfg = new Properties();
 +      zooCfg.setProperty("tickTime", "2000");
 +      zooCfg.setProperty("initLimit", "10");
 +      zooCfg.setProperty("syncLimit", "5");
 +      zooCfg.setProperty("clientPort", config.getZooKeeperPort() + "");
 +      zooCfg.setProperty("maxClientCnxns", "1000");
 +      zooCfg.setProperty("dataDir", config.getZooKeeperDir().getAbsolutePath());
 +      zooCfg.store(fileWriter, null);
 +
 +      fileWriter.close();
 +    }
 +
 +    // disable audit logging for mini....
 +    InputStream auditStream = this.getClass().getResourceAsStream("/auditLog.xml");
 +
 +    if (auditStream != null) {
 +      FileUtils.copyInputStreamToFile(auditStream, new File(config.getConfDir(), "auditLog.xml"));
 +    }
 +  }
 +
 +  private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings) throws IOException {
 +    FileWriter fileWriter = new FileWriter(file);
 +    fileWriter.append("<configuration>\n");
 +
 +    for (Entry<String,String> entry : settings) {
 +      String value = entry.getValue().replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;");
 +      fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + value + "</value></property>\n");
 +    }
 +    fileWriter.append("</configuration>\n");
 +    fileWriter.close();
 +  }
 +
 +  private void writeConfigProperties(File file, Map<String,String> settings) throws IOException {
 +    FileWriter fileWriter = new FileWriter(file);
 +
 +    for (Entry<String,String> entry : settings.entrySet())
 +      fileWriter.append(entry.getKey() + "=" + entry.getValue() + "\n");
 +    fileWriter.close();
 +  }
 +
 +  /**
 +   * Starts Accumulo and Zookeeper processes. Can only be called once.
 +   *
 +   * @throws IllegalStateException
 +   *           if already started
 +   */
 +  @Override
 +  public synchronized void start() throws IOException, InterruptedException {
 +    if (config.useExistingInstance()) {
 +      Configuration acuConf = config.getAccumuloConfiguration();
 +      Configuration hadoopConf = config.getHadoopConfiguration();
 +
 +      ConfigurationCopy cc = new ConfigurationCopy(acuConf);
 +      VolumeManager fs;
 +      try {
 +        fs = VolumeManagerImpl.get(cc, hadoopConf);
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
 +
 +      String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
 +      IZooReaderWriter zrw = new ZooReaderWriterFactory().getZooReaderWriter(cc.get(Property.INSTANCE_ZK_HOST),
 +          (int) cc.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), cc.get(Property.INSTANCE_SECRET));
 +
 +      String rootPath = ZooUtil.getRoot(instanceIdFromFile);
 +
 +      String instanceName = null;
 +      try {
 +        for (String name : zrw.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
 +          String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
 +          byte[] bytes = zrw.getData(instanceNamePath, new Stat());
-           String iid = new String(bytes, Constants.UTF8);
++          String iid = new String(bytes, UTF_8);
 +          if (iid.equals(instanceIdFromFile)) {
 +            instanceName = name;
 +          }
 +        }
 +      } catch (KeeperException e) {
 +        throw new RuntimeException("Unable to read instance name from zookeeper.", e);
 +      }
 +      if (instanceName == null)
 +        throw new RuntimeException("Unable to read instance name from zookeeper.");
 +
 +      config.setInstanceName(instanceName);
 +      if (!AccumuloStatus.isAccumuloOffline(zrw, rootPath))
 +        throw new RuntimeException("The Accumulo instance being used is already running. Aborting.");
 +    } else {
 +      if (!initialized) {
 +        Runtime.getRuntime().addShutdownHook(new Thread() {
 +          @Override
 +          public void run() {
 +            try {
 +              MiniAccumuloClusterImpl.this.stop();
 +            } catch (IOException e) {
 +              e.printStackTrace();
 +            } catch (InterruptedException e) {
 +              e.printStackTrace();
 +            }
 +          }
 +        });
 +      }
 +
 +      if (zooKeeperProcess == null) {
 +        zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
 +      }
 +
 +      if (!initialized) {
 +        // sleep a little bit to let zookeeper come up before calling init, seems to work better
 +        long startTime = System.currentTimeMillis();
 +        while (true) {
 +          Socket s = null;
 +          try {
 +            s = new Socket("localhost", config.getZooKeeperPort());
 +            s.getOutputStream().write("ruok\n".getBytes());
 +            s.getOutputStream().flush();
 +            byte buffer[] = new byte[100];
 +            int n = s.getInputStream().read(buffer);
 +            if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
 +              break;
 +          } catch (Exception e) {
 +            if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
 +              throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in "
 +                  + config.getLogDir() + " for errors.  Last exception: " + e);
 +            }
 +          } finally {
 +            if (s != null)
 +              s.close();
 +          }
 +        }
 +        Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
 +        int ret = initProcess.waitFor();
 +        if (ret != 0) {
 +          throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
 +        }
 +        initialized = true;
 +      }
 +    }
 +    
 +    log.info("Starting MAC against instance {} and zookeeper(s) {}.", config.getInstanceName(), config.getZooKeepers());
 +    
 +    synchronized (tabletServerProcesses) {
 +      for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
 +        tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
 +      }
 +    }
 +    int ret = 0;
 +    for (int i = 0; i < 5; i++) {
 +      ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()).waitFor();
 +      if (ret == 0)
 +        break;
 +      UtilWaitThread.sleep(1000);
 +    }
 +    if (ret != 0) {
 +      throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
 +    }
 +    if (masterProcess == null) {
 +      masterProcess = _exec(Master.class, ServerType.MASTER);
 +    }
 +
 +    if (gcProcess == null) {
 +      gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
 +    }
 +
 +    if (null == executor) {
 +      executor = Executors.newSingleThreadExecutor();
 +    }
 +  }
 +
 +  private List<String> buildRemoteDebugParams(int port) {
 +    return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d", port)});
 +  }
 +
 +  /**
 +   * @return generated remote debug ports if in debug mode.
 +   * @since 1.6.0
 +   */
 +  public Set<Pair<ServerType,Integer>> getDebugPorts() {
 +    return debugPorts;
 +  }
 +
 +  List<ProcessReference> references(Process... procs) {
 +    List<ProcessReference> result = new ArrayList<ProcessReference>();
 +    for (Process proc : procs) {
 +      result.add(new ProcessReference(proc));
 +    }
 +    return result;
 +  }
 +
 +  public Map<ServerType,Collection<ProcessReference>> getProcesses() {
 +    Map<ServerType,Collection<ProcessReference>> result = new HashMap<ServerType,Collection<ProcessReference>>();
 +    result.put(ServerType.MASTER, references(masterProcess));
 +    result.put(ServerType.TABLET_SERVER, references(tabletServerProcesses.toArray(new Process[0])));
 +    result.put(ServerType.ZOOKEEPER, references(zooKeeperProcess));
 +    if (null != gcProcess) {
 +      result.put(ServerType.GARBAGE_COLLECTOR, references(gcProcess));
 +    }
 +    return result;
 +  }
 +
 +  public void killProcess(ServerType type, ProcessReference proc) throws ProcessNotFoundException, InterruptedException {
 +    boolean found = false;
 +    switch (type) {
 +      case MASTER:
 +        if (proc.equals(masterProcess)) {
 +          masterProcess.destroy();
 +          masterProcess.waitFor();
 +          masterProcess = null;
 +          found = true;
 +        }
 +        break;
 +      case TABLET_SERVER:
 +        synchronized (tabletServerProcesses) {
 +          for (Process tserver : tabletServerProcesses) {
 +            if (proc.equals(tserver)) {
 +              tabletServerProcesses.remove(tserver);
 +              tserver.destroy();
 +              tserver.waitFor();
 +              found = true;
 +              break;
 +            }
 +          }
 +        }
 +        break;
 +      case ZOOKEEPER:
 +        if (proc.equals(zooKeeperProcess)) {
 +          zooKeeperProcess.destroy();
 +          zooKeeperProcess.waitFor();
 +          zooKeeperProcess = null;
 +          found = true;
 +        }
 +        break;
 +      case GARBAGE_COLLECTOR:
 +        if (proc.equals(gcProcess)) {
 +          gcProcess.destroy();
 +          gcProcess.waitFor();
 +          gcProcess = null;
 +          found = true;
 +        }
 +        break;
 +    }
 +    if (!found)
 +      throw new ProcessNotFoundException();
 +  }
 +
 +  /**
 +   * @return Accumulo instance name
 +   */
 +  @Override
 +  public String getInstanceName() {
 +    return config.getInstanceName();
 +  }
 +
 +  /**
 +   * @return zookeeper connection string
 +   */
 +  @Override
 +  public String getZooKeepers() {
 +    return config.getZooKeepers();
 +  }
 +
 +  /**
 +   * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. However its probably best to
 +   * call stop in a finally block as soon as possible.
 +   */
 +  @Override
 +  public synchronized void stop() throws IOException, InterruptedException {
 +    if (null == executor) {
 +      // keep repeated calls to stop() from failing
 +      return;
 +    }
 +
 +    for (LogWriter lw : logWriters) {
 +      lw.flush();
 +    }
 +
 +    if (gcProcess != null) {
 +      try {
 +        stopProcessWithTimeout(gcProcess, 30, TimeUnit.SECONDS);
 +      } catch (ExecutionException e) {
 +        log.warn("GarbageCollector did not fully stop after 30 seconds", e);
 +      } catch (TimeoutException e) {
 +        log.warn("GarbageCollector did not fully stop after 30 seconds", e);
 +      }
 +    }
 +    if (masterProcess != null) {
 +      try {
 +        stopProcessWithTimeout(masterProcess, 30, TimeUnit.SECONDS);
 +      } catch (ExecutionException e) {
 +        log.warn("Master did not fully stop after 30 seconds", e);
 +      } catch (TimeoutException e) {
 +        log.warn("Master did not fully stop after 30 seconds", e);
 +      }
 +    }
 +    if (tabletServerProcesses != null) {
 +      synchronized (tabletServerProcesses) {
 +        for (Process tserver : tabletServerProcesses) {
 +          try {
 +            stopProcessWithTimeout(tserver, 30, TimeUnit.SECONDS);
 +          } catch (ExecutionException e) {
 +            log.warn("TabletServer did not fully stop after 30 seconds", e);
 +          } catch (TimeoutException e) {
 +            log.warn("TabletServer did not fully stop after 30 seconds", e);
 +          }
 +        }
 +      }
 +    }
 +    if (zooKeeperProcess != null) {
 +      try {
 +        stopProcessWithTimeout(zooKeeperProcess, 30, TimeUnit.SECONDS);
 +      } catch (ExecutionException e) {
 +        log.warn("ZooKeeper did not fully stop after 30 seconds", e);
 +      } catch (TimeoutException e) {
 +        log.warn("ZooKeeper did not fully stop after 30 seconds", e);
 +      }
 +    }
 +
 +    zooKeeperProcess = null;
 +    masterProcess = null;
 +    gcProcess = null;
 +    tabletServerProcesses.clear();
 +
 +    // ACCUMULO-2985 stop the ExecutorService after we finished using it to stop accumulo procs
 +    if (null != executor) {
 +      List<Runnable> tasksRemaining = executor.shutdownNow();
 +
 +      // the single thread executor shouldn't have any pending tasks, but check anyways
 +      if (!tasksRemaining.isEmpty()) {
 +        log.warn("Unexpectedly had " + tasksRemaining.size() + " task(s) remaining in threadpool for execution when being stopped");
 +      }
 +
 +      executor = null;
 +    }
 +
 +    if (config.useMiniDFS() && miniDFS != null)
 +      miniDFS.shutdown();
 +    for (Process p : cleanup) {
 +      p.destroy();
 +      p.waitFor();
 +    }
 +    miniDFS = null;
 +  }
 +
 +  /**
 +   * @since 1.6.0
 +   */
 +  @Override
 +  public MiniAccumuloConfigImpl getConfig() {
 +    return config;
 +  }
 +
 +  /**
 +   * Utility method to get a connector to the MAC.
 +   *
 +   * @since 1.6.0
 +   */
 +  @Override
 +  public Connector getConnector(String user, String passwd) throws AccumuloException, AccumuloSecurityException {
 +    Instance instance = new ZooKeeperInstance(getClientConfig());
 +    return instance.getConnector(user, new PasswordToken(passwd));
 +  }
 +
 +  @Override
 +  public ClientConfiguration getClientConfig() {
 +    return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(
 +        this.getZooKeepers());
 +  }
 +
 +  public FileSystem getFileSystem() {
 +    try {
 +      return FileSystem.get(new URI(dfsUri), new Configuration());
 +    } catch (Exception e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  // Visible for testing
 +  protected void setShutdownExecutor(ExecutorService svc) {
 +    this.executor = svc;
 +  }
 +
 +  // Visible for testing
 +  protected ExecutorService getShutdownExecutor() {
 +    return executor;
 +  }
 +
 +  private int stopProcessWithTimeout(final Process proc, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
 +    FutureTask<Integer> future = new FutureTask<Integer>(new Callable<Integer>() {
 +      @Override
 +      public Integer call() throws InterruptedException {
 +        proc.destroy();
 +        return proc.waitFor();
 +      }
 +    });
 +
 +    executor.execute(future);
 +
 +    return future.get(timeout, unit);
 +  }
 +
 +  /**
 +   * Get programmatic interface to information available in a normal monitor. XXX the returned structure won't contain information about the metadata table
 +   * until there is data in it. e.g. if you want to see the metadata table you should create a table.
 +   * 
 +   * @since 1.6.1
 +   */
 +  public MasterMonitorInfo getMasterMonitorInfo() throws AccumuloException, AccumuloSecurityException {
 +    MasterClientService.Iface client = null;
 +    MasterMonitorInfo stats = null;
 +    try {
 +      Instance instance = new ZooKeeperInstance(getClientConfig());
 +      client = MasterClient.getConnectionWithRetry(instance);
 +      stats = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get(instance).toThrift(instance));
 +    } catch (ThriftSecurityException exception) {
 +      throw new AccumuloSecurityException(exception);
 +    } catch (TException exception) {
 +      throw new AccumuloException(exception);
 +    } finally {
 +      if (client != null) {
 +        MasterClient.close(client);
 +      }
 +    }
 +    return stats;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
index be29dbb,48e2906..30da0cba
--- a/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
@@@ -16,6 -16,6 +16,8 @@@
   */
  package org.apache.accumulo.proxy;
  
++import static com.google.common.base.Charsets.UTF_8;
++
  import java.nio.ByteBuffer;
  import java.util.ArrayList;
  import java.util.Collection;
@@@ -194,13 -172,11 +195,13 @@@ public class ProxyServer implements Acc
    }
    
    protected Connector getConnector(ByteBuffer login) throws Exception {
-     String[] pair = new String(login.array(), login.position(), login.remaining(), Constants.UTF8).split(",", 2);
 -    TCredentials user = CredentialHelper.fromByteArray(ByteBufferUtil.toBytes(login));
 -    if (user == null)
 -      throw new org.apache.accumulo.proxy.thrift.AccumuloSecurityException("unknown user");
 -    Connector connector = instance.getConnector(user.getPrincipal(), CredentialHelper.extractToken(user));
 -    return connector;
++    String[] pair = new String(login.array(), login.position(), login.remaining(), UTF_8).split(",", 2);
 +    if (instance.getInstanceID().equals(pair[0])) {
 +      Credentials creds = Credentials.deserialize(pair[1]);
 +      return instance.getConnector(creds.getPrincipal(), creds.getToken());
 +    } else {
 +      throw new org.apache.accumulo.core.client.AccumuloSecurityException(pair[0], org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.INVALID_INSTANCEID);
 +    }
    }
    
    private void handleAccumuloException(AccumuloException e) throws org.apache.accumulo.proxy.thrift.TableNotFoundException,
@@@ -1489,7 -1432,8 +1490,7 @@@
    public ByteBuffer login(String principal, Map<String,String> loginProperties) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException {
      try {
        AuthenticationToken token = getToken(principal, loginProperties);
-       ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(Constants.UTF8));
 -      TCredentials credential = CredentialHelper.create(principal, token, instance.getInstanceID());
 -      ByteBuffer login = ByteBuffer.wrap(CredentialHelper.asByteArray(credential));
++      ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(UTF_8));
        getConnector(login); // check to make sure user exists
        return login;
      } catch (AccumuloSecurityException e) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------


[37/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 1e7658a,0000000..6c40094
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -1,329 -1,0 +1,331 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileInputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.net.InetAddress;
 +import java.net.UnknownHostException;
 +import java.util.Map.Entry;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.trace.DistributedTrace;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.util.Version;
 +import org.apache.accumulo.core.volume.Volume;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.ReadOnlyStore;
 +import org.apache.accumulo.fate.ReadOnlyTStore;
 +import org.apache.accumulo.fate.ZooStore;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.watcher.Log4jConfiguration;
 +import org.apache.accumulo.server.watcher.MonitorLog4jWatcher;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.helpers.LogLog;
 +import org.apache.log4j.xml.DOMConfigurator;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class Accumulo {
 +
 +  private static final Logger log = Logger.getLogger(Accumulo.class);
 +
 +  public static synchronized void updateAccumuloVersion(VolumeManager fs, int oldVersion) {
 +    for (Volume volume : fs.getVolumes()) {
 +      try {
 +        if (getAccumuloPersistentVersion(fs) == oldVersion) {
 +          log.debug("Attempting to upgrade " + volume);
 +          Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
 +          fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
 +          // TODO document failure mode & recovery if FS permissions cause above to work and below to fail ACCUMULO-2596
 +          Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(oldVersion));
 +          if (!fs.delete(prevDataVersionLoc)) {
 +            throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
 +          }
 +        }
 +      } catch (IOException e) {
 +        throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
 +      }
 +    }
 +  }
 +
 +  public static synchronized int getAccumuloPersistentVersion(FileSystem fs, Path path) {
 +    int dataVersion;
 +    try {
 +      FileStatus[] files = fs.listStatus(path);
 +      if (files == null || files.length == 0) {
 +        dataVersion = -1; // assume it is 0.5 or earlier
 +      } else {
 +        dataVersion = Integer.parseInt(files[0].getPath().getName());
 +      }
 +      return dataVersion;
 +    } catch (IOException e) {
 +      throw new RuntimeException("Unable to read accumulo version: an error occurred.", e);
 +    }
 +  }
 +
 +  public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
 +    // It doesn't matter which Volume is used as they should all have the data version stored
 +    Volume v = fs.getVolumes().iterator().next();
 +    Path path = ServerConstants.getDataVersionLocation(v);
 +    return getAccumuloPersistentVersion(v.getFileSystem(), path);
 +  }
 +
 +  public static synchronized Path getAccumuloInstanceIdPath(VolumeManager fs) {
 +    // It doesn't matter which Volume is used as they should all have the instance ID stored
 +    Volume v = fs.getVolumes().iterator().next();
 +    return ServerConstants.getInstanceIdLocation(v);
 +  }
 +
 +  public static void enableTracing(String address, String application) {
 +    try {
 +      DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address);
 +    } catch (Exception ex) {
 +      log.error("creating remote sink for trace spans", ex);
 +    }
 +  }
 +
 +  /**
 +   * Finds the best log4j configuration file. A generic file is used only if an
 +   * application-specific file is not available. An XML file is preferred over
 +   * a properties file, if possible.
 +   *
 +   * @param confDir directory where configuration files should reside
 +   * @param application application name for configuration file name
 +   * @return configuration file name
 +   */
 +  static String locateLogConfig(String confDir, String application) {
 +    String explicitConfigFile = System.getProperty("log4j.configuration");
 +    if (explicitConfigFile != null) {
 +      return explicitConfigFile;
 +    }
 +    String[] configFiles = {
 +      String.format("%s/%s_logger.xml", confDir, application),
 +      String.format("%s/%s_logger.properties", confDir, application),
 +      String.format("%s/generic_logger.xml", confDir),
 +      String.format("%s/generic_logger.properties", confDir)
 +    };
 +    String defaultConfigFile = configFiles[2];  // generic_logger.xml
 +    for (String f : configFiles) {
 +      if (new File(f).exists()) {
 +        return f;
 +      }
 +    }
 +    return defaultConfigFile;
 +  }
 +
 +  public static void setupLogging(String application) throws UnknownHostException {
 +    System.setProperty("org.apache.accumulo.core.application", application);
 +
 +    if (System.getenv("ACCUMULO_LOG_DIR") != null)
 +      System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_LOG_DIR"));
 +    else
 +      System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_HOME") + "/logs/");
 +
 +    String localhost = InetAddress.getLocalHost().getHostName();
 +    System.setProperty("org.apache.accumulo.core.ip.localhost.hostname", localhost);
 +
 +    // Use a specific log config, if it exists
 +    String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
 +    // Turn off messages about not being able to reach the remote logger... we protect against that.
 +    LogLog.setQuietMode(true);
 +
 +    // Read the auditing config
 +    String auditConfig = String.format("%s/auditLog.xml", System.getenv("ACCUMULO_CONF_DIR"));
 +
 +    // Set up local file-based logging right away
 +    Log4jConfiguration logConf = new Log4jConfiguration(logConfigFile);
 +    logConf.resetLogger();
 +
 +    // Watch the auditLog.xml for the future updates
 +    DOMConfigurator.configureAndWatch(auditConfig, 5000);
 +  }
 +
 +  public static void init(VolumeManager fs, ServerConfiguration serverConfig, String application) throws IOException {
 +    final AccumuloConfiguration conf = serverConfig.getConfiguration();
 +    final Instance instance = serverConfig.getInstance();
 +
 +    // Use a specific log config, if it exists
 +    final String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
 +
 +    // Set up polling log4j updates and log-forwarding using information advertised in zookeeper by the monitor
 +    MonitorLog4jWatcher logConfigWatcher = new MonitorLog4jWatcher(instance.getInstanceID(), logConfigFile);
 +    logConfigWatcher.setDelay(5000L);
 +    logConfigWatcher.start();
 +
 +    // Makes sure the log-forwarding to the monitor is configured
 +    int logPort = conf.getPort(Property.MONITOR_LOG4J_PORT);
 +    System.setProperty("org.apache.accumulo.core.host.log.port", Integer.toString(logPort));
 +
 +    log.info(application + " starting");
 +    log.info("Instance " + serverConfig.getInstance().getInstanceID());
 +    int dataVersion = Accumulo.getAccumuloPersistentVersion(fs);
 +    log.info("Data Version " + dataVersion);
 +    Accumulo.waitForZookeeperAndHdfs(fs);
 +
 +    Version codeVersion = new Version(Constants.VERSION);
 +    if (!(canUpgradeFromDataVersion(dataVersion))) {
 +      throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible with files stored using data version " + dataVersion);
 +    }
 +
 +    TreeMap<String,String> sortedProps = new TreeMap<String,String>();
 +    for (Entry<String,String> entry : conf)
 +      sortedProps.put(entry.getKey(), entry.getValue());
 +
 +    for (Entry<String,String> entry : sortedProps.entrySet()) {
 +      String key = entry.getKey();
 +      log.info(key + " = " + (Property.isSensitive(key) ? "<hidden>" : entry.getValue()));
 +    }
 +
 +    monitorSwappiness();
 +  }
 +
 +  /**
 +   * Sanity check that the current persistent version is allowed to upgrade to the version of Accumulo running.
 +   * @param dataVersion the version that is persisted in the backing Volumes
 +   */
 +  public static boolean canUpgradeFromDataVersion(final int dataVersion) {
 +    return dataVersion == ServerConstants.DATA_VERSION || dataVersion == ServerConstants.PREV_DATA_VERSION || dataVersion == ServerConstants.TWO_DATA_VERSIONS_AGO;
 +  }
 +
 +  /**
 +   * Does the data version number stored in the backing Volumes indicate we need to upgrade something?
 +   */
 +  public static boolean persistentVersionNeedsUpgrade(final int accumuloPersistentVersion) {
 +    return accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO || accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION;
 +  }
 +
 +  /**
 +   *
 +   */
 +  public static void monitorSwappiness() {
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        try {
 +          String procFile = "/proc/sys/vm/swappiness";
 +          File swappiness = new File(procFile);
 +          if (swappiness.exists() && swappiness.canRead()) {
 +            InputStream is = new FileInputStream(procFile);
 +            try {
 +              byte[] buffer = new byte[10];
 +              int bytes = is.read(buffer);
-               String setting = new String(buffer, 0, bytes, Constants.UTF8);
++              String setting = new String(buffer, 0, bytes, UTF_8);
 +              setting = setting.trim();
 +              if (bytes > 0 && Integer.parseInt(setting) > 10) {
 +                log.warn("System swappiness setting is greater than ten (" + setting + ") which can cause time-sensitive operations to be delayed. "
 +                    + " Accumulo is time sensitive because it needs to maintain distributed lock agreement.");
 +              }
 +            } finally {
 +              is.close();
 +            }
 +          }
 +        } catch (Throwable t) {
 +          log.error(t, t);
 +        }
 +      }
 +    }, 1000, 10 * 60 * 1000);
 +  }
 +
 +  public static void waitForZookeeperAndHdfs(VolumeManager fs) {
 +    log.info("Attempting to talk to zookeeper");
 +    while (true) {
 +      try {
 +        ZooReaderWriter.getInstance().getChildren(Constants.ZROOT);
 +        break;
 +      } catch (InterruptedException e) {
 +        // ignored
 +      } catch (KeeperException ex) {
 +        log.info("Waiting for accumulo to be initialized");
 +        UtilWaitThread.sleep(1000);
 +      }
 +    }
 +    log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
 +    long sleep = 1000;
 +    int unknownHostTries = 3;
 +    while (true) {
 +      try {
 +        if (fs.isReady())
 +          break;
 +        log.warn("Waiting for the NameNode to leave safemode");
 +      } catch (IOException ex) {
 +        log.warn("Unable to connect to HDFS", ex);
 +      } catch (IllegalArgumentException exception) {
 +        /* Unwrap the UnknownHostException so we can deal with it directly */
 +        if (exception.getCause() instanceof UnknownHostException) {
 +          if (unknownHostTries > 0) {
 +            log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause());
 +            /* We need to make sure our sleep period is long enough to avoid getting a cached failure of the host lookup. */
 +            sleep = Math.max(sleep, (AddressUtil.getAddressCacheNegativeTtl((UnknownHostException)(exception.getCause()))+1)*1000);
 +          } else {
 +            log.error("Unable to connect to HDFS and have exceeded max number of retries.", exception);
 +            throw exception;
 +          }
 +          unknownHostTries--;
 +        } else {
 +          throw exception;
 +        }
 +      }
 +      log.info("Backing off due to failure; current sleep period is " + sleep / 1000. + " seconds");
 +      UtilWaitThread.sleep(sleep);
 +      /* Back off to give transient failures more time to clear. */
 +      sleep = Math.min(60 * 1000, sleep * 2);
 +    }
 +    log.info("Connected to HDFS");
 +  }
 +
 +  /**
 +   * Exit loudly if there are outstanding Fate operations.
 +   * Since Fate serializes class names, we need to make sure there are no queued
 +   * transactions from a previous version before continuing an upgrade. The status of the operations is
 +   * irrelevant; those in SUCCESSFUL status cause the same problem as those just queued.
 +   *
 +   * Note that the Master should not allow write access to Fate until after all upgrade steps are complete.
 +   *
 +   * Should be called as a guard before performing any upgrade steps, after determining that an upgrade is needed.
 +   *
 +   * see ACCUMULO-2519
 +   */
 +  public static void abortIfFateTransactions() {
 +    try {
 +      final ReadOnlyTStore<Accumulo> fate = new ReadOnlyStore<Accumulo>(new ZooStore<Accumulo>(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZFATE,
 +          ZooReaderWriter.getInstance()));
 +      if (!(fate.list().isEmpty())) {
 +        throw new AccumuloException("Aborting upgrade because there are outstanding FATE transactions from a previous Accumulo version. Please see the README document for instructions on what to do under your previous version.");
 +      }
 +    } catch (Exception exception) {
 +      log.fatal("Problem verifying Fate readiness", exception);
 +      System.exit(1);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 3508164,0000000..05f3c6c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -1,198 -1,0 +1,200 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.client;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.nio.ByteBuffer;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.impl.ConnectorImpl;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.util.ByteBufferUtil;
 +import org.apache.accumulo.core.util.OpTimer;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * An implementation of Instance that looks in HDFS and ZooKeeper to find the master and root tablet location.
 + * 
 + */
 +public class HdfsZooInstance implements Instance {
 +
 +  private HdfsZooInstance() {
 +    AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
 +    zooCache = new ZooCacheFactory().getZooCache(acuConf.get(Property.INSTANCE_ZK_HOST), (int) acuConf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
 +  }
 +
 +  private static HdfsZooInstance cachedHdfsZooInstance = null;
 +
 +  public static synchronized Instance getInstance() {
 +    if (cachedHdfsZooInstance == null)
 +      cachedHdfsZooInstance = new HdfsZooInstance();
 +    return cachedHdfsZooInstance;
 +  }
 +
 +  private static ZooCache zooCache;
 +  private static String instanceId = null;
 +  private static final Logger log = Logger.getLogger(HdfsZooInstance.class);
 +
 +  @Override
 +  public String getRootTabletLocation() {
 +    String zRootLocPath = ZooUtil.getRoot(this) + RootTable.ZROOT_TABLET_LOCATION;
 +
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zoocache.");
 +
 +    byte[] loc = zooCache.get(zRootLocPath);
 +
-     opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++    opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
 +
 +    if (loc == null) {
 +      return null;
 +    }
 +
-     return new String(loc, Constants.UTF8).split("\\|")[0];
++    return new String(loc, UTF_8).split("\\|")[0];
 +  }
 +
 +  @Override
 +  public List<String> getMasterLocations() {
 +
 +    String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
 +
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
 +
 +    byte[] loc = ZooLock.getLockData(zooCache, masterLocPath, null);
 +
-     opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++    opTimer.stop("Found master at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
 +
 +    if (loc == null) {
 +      return Collections.emptyList();
 +    }
 +
-     return Collections.singletonList(new String(loc, Constants.UTF8));
++    return Collections.singletonList(new String(loc, UTF_8));
 +  }
 +
 +  @Override
 +  public String getInstanceID() {
 +    if (instanceId == null)
 +      _getInstanceID();
 +    return instanceId;
 +  }
 +
 +  private static synchronized void _getInstanceID() {
 +    if (instanceId == null) {
 +      AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
 +      // InstanceID should be the same across all volumes, so just choose one
 +      VolumeManager fs;
 +      try {
 +        fs = VolumeManagerImpl.get();
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
 +      log.trace("Looking for instanceId from " + instanceIdPath);
 +      String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
 +      instanceId = instanceIdFromFile;
 +    }
 +  }
 +
 +  @Override
 +  public String getInstanceName() {
 +    return ZooKeeperInstance.lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
 +  }
 +
 +  @Override
 +  public String getZooKeepers() {
 +    return ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
 +  }
 +
 +  @Override
 +  public int getZooKeepersSessionTimeOut() {
 +    return (int) ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
 +  }
 +
 +  @Override
 +  public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
 +    return new ConnectorImpl(this, new Credentials(principal, token));
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
 +    return getConnector(user, new PasswordToken(pass));
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
 +    return getConnector(user, ByteBufferUtil.toBytes(pass));
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
 +    return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
 +  }
 +
 +  private AccumuloConfiguration conf = null;
 +
 +  @Deprecated
 +  @Override
 +  public AccumuloConfiguration getConfiguration() {
 +    if (conf == null)
 +      conf = new ServerConfiguration(this).getConfiguration();
 +    return conf;
 +  }
 +
 +  @Override
 +  @Deprecated
 +  public void setConfiguration(AccumuloConfiguration conf) {
 +    this.conf = conf;
 +  }
 +
 +  public static void main(String[] args) {
 +    Instance instance = HdfsZooInstance.getInstance();
 +    System.out.println("Instance Name: " + instance.getInstanceName());
 +    System.out.println("Instance ID: " + instance.getInstanceID());
 +    System.out.println("ZooKeepers: " + instance.getZooKeepers());
 +    System.out.println("Masters: " + StringUtil.join(instance.getMasterLocations(), ", "));
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
index df1be50,0000000..18feb4f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
@@@ -1,160 -1,0 +1,161 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.List;
 +import java.util.Map;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * A helper object for accessing properties in a {@link ZooCache}.
 + */
 +public class ZooCachePropertyAccessor {
 +  private static final Logger log = Logger.getLogger(ZooCachePropertyAccessor.class);
 +
 +  static class PropCacheKey {
 +    final String instanceId;
 +    final String scope;
 +
 +    PropCacheKey(String iid, String s) {
 +      instanceId = iid;
 +      scope = s;
 +    }
 +
 +    @Override
 +    public boolean equals(Object other) {
 +      if (this == other) {
 +        return true;
 +      }
 +      if (!(other instanceof PropCacheKey)) {
 +        return false;
 +      }
 +      PropCacheKey o = (PropCacheKey) other;
 +      return (instanceId.equals(o.instanceId) && scope.equals(o.scope));
 +    }
 +
 +    @Override
 +    public int hashCode() {
 +      int c = 17;
 +      c = (37 * c) + instanceId.hashCode();
 +      c = (37 * c) + scope.hashCode();
 +      return c;
 +    }
 +  }
 +
 +  private final ZooCache propCache;
 +
 +  /**
 +   * Creates a new accessor.
 +   *
 +   * @param propCache
 +   *          property cache
 +   */
 +  ZooCachePropertyAccessor(ZooCache propCache) {
 +    this.propCache = propCache;
 +  }
 +
 +  /**
 +   * Gets the property cache accessed by this object.
 +   *
 +   * @return property cache
 +   */
 +  ZooCache getZooCache() {
 +    return propCache;
 +  }
 +
 +  /**
 +   * Gets a property. If the property is not in ZooKeeper or is present but an invalid format for the property type, the parent configuration is consulted (if
 +   * provided).
 +   *
 +   * @param property
 +   *          property to get
 +   * @param path
 +   *          ZooKeeper path where properties lie
 +   * @param parent
 +   *          parent configuration (optional)
 +   * @return property value, or null if not found
 +   */
 +  String get(Property property, String path, AccumuloConfiguration parent) {
 +    String key = property.getKey();
 +    String value = get(path + "/" + key);
 +
 +    if (value == null || !property.getType().isValidFormat(value)) {
 +      if (value != null) {
 +        log.error("Using default value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
 +      }
 +      if (parent != null) {
 +        value = parent.get(property);
 +      }
 +    }
 +    return value;
 +  }
 +
 +  private String get(String path) {
 +    byte[] v = propCache.get(path);
 +    if (v != null) {
-       return new String(v, Constants.UTF8);
++      return new String(v, UTF_8);
 +    } else {
 +      return null;
 +    }
 +  }
 +
 +  /**
 +   * Gets all properties into the given map. Properties are filtered using the given filter. Properties from a parent configuration are also added to the map
 +   * and filtered, either using a separate filter or, if not specified, the other filter.
 +   *
 +   * @param props
 +   *          map to populate with properties
 +   * @param path
 +   *          ZooKeeper path where properties lie
 +   * @param filter
 +   *          property filter (required)
 +   * @param parent
 +   *          parent configuration (required)
 +   * @param parentFilter
 +   *          separate filter for parent properties (optional)
 +   */
 +  void getProperties(Map<String,String> props, String path, PropertyFilter filter, AccumuloConfiguration parent, PropertyFilter parentFilter) {
 +    parent.getProperties(props, parentFilter != null ? parentFilter : filter);
 +
 +    List<String> children = propCache.getChildren(path);
 +    if (children != null) {
 +      for (String child : children) {
 +        if (child != null && filter.accept(child)) {
 +          String value = get(path + "/" + child);
 +          if (value != null) {
 +            props.put(child, value);
 +          }
 +        }
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Clears the internal {@link ZooCache}.
 +   */
 +  void invalidateCache() {
 +    propCache.clear();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index a49ec43,0000000..6d25f1d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@@ -1,118 -1,0 +1,120 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.log4j.Logger;
 +
 +public class ZooConfiguration extends AccumuloConfiguration {
 +  private static final Logger log = Logger.getLogger(ZooConfiguration.class);
 +
 +  private final String instanceId;
 +  private final ZooCache propCache;
 +  private final AccumuloConfiguration parent;
 +  private final Map<String,String> fixedProps = Collections.synchronizedMap(new HashMap<String,String>());
 +
 +  protected ZooConfiguration(String instanceId, ZooCache propCache, AccumuloConfiguration parent) {
 +    this.instanceId = instanceId;
 +    this.propCache = propCache;
 +    this.parent = parent;
 +  }
 +
 +  @Override
 +  public void invalidateCache() {
 +    if (propCache != null)
 +      propCache.clear();
 +  }
 +
 +  /**
 +   * Gets the parent configuration of this configuration.
 +   *
 +   * @return parent configuration
 +   */
 +  public AccumuloConfiguration getParentConfiguration() {
 +    return parent;
 +  }
 +
 +  private String _get(Property property) {
 +    String key = property.getKey();
 +    String value = null;
 +
 +    if (Property.isValidZooPropertyKey(key)) {
 +      value = get(key);
 +    }
 +
 +    if (value == null || !property.getType().isValidFormat(value)) {
 +      if (value != null)
 +        log.error("Using parent value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
 +      value = parent.get(property);
 +    }
 +    return value;
 +  }
 +
 +  @Override
 +  public String get(Property property) {
 +    if (Property.isFixedZooPropertyKey(property)) {
 +      if (fixedProps.containsKey(property.getKey())) {
 +        return fixedProps.get(property.getKey());
 +      } else {
 +        synchronized (fixedProps) {
 +          String val = _get(property);
 +          fixedProps.put(property.getKey(), val);
 +          return val;
 +        }
 +
 +      }
 +    } else {
 +      return _get(property);
 +    }
 +  }
 +
 +  private String get(String key) {
 +    String zPath = ZooUtil.getRoot(instanceId) + Constants.ZCONFIG + "/" + key;
 +    byte[] v = propCache.get(zPath);
 +    String value = null;
 +    if (v != null)
-       value = new String(v, Constants.UTF8);
++      value = new String(v, UTF_8);
 +    return value;
 +  }
 +
 +  @Override
 +  public void getProperties(Map<String,String> props, PropertyFilter filter) {
 +    parent.getProperties(props, filter);
 +
 +    List<String> children = propCache.getChildren(ZooUtil.getRoot(instanceId) + Constants.ZCONFIG);
 +    if (children != null) {
 +      for (String child : children) {
 +        if (child != null && filter.accept(child)) {
 +          String value = get(child);
 +          if (value != null)
 +            props.put(child, value);
 +        }
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 807e821,0000000..587e38a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -1,315 -1,0 +1,317 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.constraints;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.HashSet;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.constraints.Constraint;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +public class MetadataConstraints implements Constraint {
 +  
 +  private ZooCache zooCache = null;
 +  private String zooRoot = null;
 +  
 +  private static final Logger log = Logger.getLogger(MetadataConstraints.class);
 +  
 +  private static boolean[] validTableNameChars = new boolean[256];
 +  
 +  {
 +    for (int i = 0; i < 256; i++) {
 +      validTableNameChars[i] = ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) || i == '!';
 +    }
 +  }
 +  
 +  private static final HashSet<ColumnFQ> validColumnQuals = new HashSet<ColumnFQ>(Arrays.asList(new ColumnFQ[] {
 +      TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN,
 +      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN,
 +      TabletsSection.ServerColumnFamily.LOCK_COLUMN, TabletsSection.ServerColumnFamily.FLUSH_COLUMN, TabletsSection.ServerColumnFamily.COMPACT_COLUMN}));
 +  
 +  private static final HashSet<Text> validColumnFams = new HashSet<Text>(Arrays.asList(new Text[] {TabletsSection.BulkFileColumnFamily.NAME,
 +      LogColumnFamily.NAME, ScanFileColumnFamily.NAME, DataFileColumnFamily.NAME,
 +      TabletsSection.CurrentLocationColumnFamily.NAME, TabletsSection.LastLocationColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME,
 +      ChoppedColumnFamily.NAME, ClonedColumnFamily.NAME}));
 +  
 +  private static boolean isValidColumn(ColumnUpdate cu) {
 +    
 +    if (validColumnFams.contains(new Text(cu.getColumnFamily())))
 +      return true;
 +    
 +    if (validColumnQuals.contains(new ColumnFQ(cu)))
 +      return true;
 +    
 +    return false;
 +  }
 +  
 +  static private ArrayList<Short> addViolation(ArrayList<Short> lst, int violation) {
 +    if (lst == null)
 +      lst = new ArrayList<Short>();
 +    lst.add((short) violation);
 +    return lst;
 +  }
 +  
 +  static private ArrayList<Short> addIfNotPresent(ArrayList<Short> lst, int intViolation) {
 +    if (lst == null)
 +      return addViolation(lst, intViolation);
 +    short violation = (short) intViolation;
 +    if (!lst.contains(violation))
 +      return addViolation(lst, intViolation);
 +    return lst;
 +  }
 +  
 +  @Override
 +  public List<Short> check(Environment env, Mutation mutation) {
 +    
 +    ArrayList<Short> violations = null;
 +    
 +    Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
 +    
 +    // check the row, it should contains at least one ; or end with <
 +    boolean containsSemiC = false;
 +    
 +    byte[] row = mutation.getRow();
 +    
 +    // always allow rows that fall within reserved areas
 +    if (row.length > 0 && row[0] == '~')
 +      return null;
 +    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
 +      return null;
 +    
 +    for (byte b : row) {
 +      if (b == ';') {
 +        containsSemiC = true;
 +      }
 +      
 +      if (b == ';' || b == '<')
 +        break;
 +      
 +      if (!validTableNameChars[0xff & b]) {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    }
 +    
 +    if (!containsSemiC) {
 +      // see if last row char is <
 +      if (row.length == 0 || row[row.length - 1] != '<') {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    } else {
 +      if (row.length == 0) {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    }
 +    
 +    if (row.length > 0 && row[0] == '!') {
 +      if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    }
 +    
 +    // ensure row is not less than Constants.METADATA_TABLE_ID
 +    if (new Text(row).compareTo(new Text(MetadataTable.ID)) < 0) {
 +      violations = addViolation(violations, 5);
 +    }
 +    
 +    boolean checkedBulk = false;
 +    
 +    for (ColumnUpdate columnUpdate : colUpdates) {
 +      Text columnFamily = new Text(columnUpdate.getColumnFamily());
 +      
 +      if (columnUpdate.isDeleted()) {
 +        if (!isValidColumn(columnUpdate)) {
 +          violations = addViolation(violations, 2);
 +        }
 +        continue;
 +      }
 +      
 +      if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
 +        violations = addViolation(violations, 6);
 +      }
 +      
 +      if (columnFamily.equals(DataFileColumnFamily.NAME)) {
 +        try {
 +          DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
 +          
 +          if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
 +            violations = addViolation(violations, 1);
 +          }
 +        } catch (NumberFormatException nfe) {
 +          violations = addViolation(violations, 1);
 +        } catch (ArrayIndexOutOfBoundsException aiooe) {
 +          violations = addViolation(violations, 1);
 +        }
 +      } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
 +        
 +      } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
 +        if (!columnUpdate.isDeleted() && !checkedBulk) {
 +          // splits, which also write the time reference, are allowed to write this reference even when
 +          // the transaction is not running because the other half of the tablet is holding a reference
 +          // to the file.
 +          boolean isSplitMutation = false;
 +          // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
 +          // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
 +          // See ACCUMULO-1230.
 +          boolean isLocationMutation = false;
 +          
 +          HashSet<Text> dataFiles = new HashSet<Text>();
 +          HashSet<Text> loadedFiles = new HashSet<Text>();
 +
-           String tidString = new String(columnUpdate.getValue(), Constants.UTF8);
++          String tidString = new String(columnUpdate.getValue(), UTF_8);
 +          int otherTidCount = 0;
 +          
 +          for (ColumnUpdate update : mutation.getUpdates()) {
 +            if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
 +              isSplitMutation = true;
 +            } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
 +              isLocationMutation = true;
 +            } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
 +              dataFiles.add(new Text(update.getColumnQualifier()));
 +            } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
 +              loadedFiles.add(new Text(update.getColumnQualifier()));
 +              
-               if (!new String(update.getValue(), Constants.UTF8).equals(tidString)) {
++              if (!new String(update.getValue(), UTF_8).equals(tidString)) {
 +                otherTidCount++;
 +              }
 +            }
 +          }
 +          
 +          if (!isSplitMutation && !isLocationMutation) {
 +            long tid = Long.parseLong(tidString);
 +            
 +            try {
 +              if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
 +                violations = addViolation(violations, 8);
 +              }
 +            } catch (Exception ex) {
 +              violations = addViolation(violations, 8);
 +            }
 +          }
 +          
 +          checkedBulk = true;
 +        }
 +      } else {
 +        if (!isValidColumn(columnUpdate)) {
 +          violations = addViolation(violations, 2);
 +        } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0
 +            && (violations == null || !violations.contains((short) 4))) {
 +          KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
 +          
 +          Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
 +          
 +          boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
 +          
 +          if (!prevEndRowLessThanEndRow) {
 +            violations = addViolation(violations, 3);
 +          }
 +        } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
 +          if (zooCache == null) {
 +            zooCache = new ZooCache();
 +          }
 +          
 +          if (zooRoot == null) {
 +            zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
 +          }
 +          
 +          boolean lockHeld = false;
-           String lockId = new String(columnUpdate.getValue(), Constants.UTF8);
++          String lockId = new String(columnUpdate.getValue(), UTF_8);
 +          
 +          try {
 +            lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
 +          } catch (Exception e) {
 +            log.debug("Failed to verify lock was held " + lockId + " " + e.getMessage());
 +          }
 +          
 +          if (!lockHeld) {
 +            violations = addViolation(violations, 7);
 +          }
 +        }
 +        
 +      }
 +    }
 +    
 +    if (violations != null) {
-       log.debug("violating metadata mutation : " + new String(mutation.getRow(), Constants.UTF8));
++      log.debug("violating metadata mutation : " + new String(mutation.getRow(), UTF_8));
 +      for (ColumnUpdate update : mutation.getUpdates()) {
-         log.debug(" update: " + new String(update.getColumnFamily(), Constants.UTF8) + ":" + new String(update.getColumnQualifier(), Constants.UTF8) + " value "
-             + (update.isDeleted() ? "[delete]" : new String(update.getValue(), Constants.UTF8)));
++        log.debug(" update: " + new String(update.getColumnFamily(), UTF_8) + ":" + new String(update.getColumnQualifier(), UTF_8) + " value "
++            + (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
 +      }
 +    }
 +    
 +    return violations;
 +  }
 +  
 +  protected Arbitrator getArbitrator() {
 +    return new ZooArbitrator();
 +  }
 +  
 +  @Override
 +  public String getViolationDescription(short violationCode) {
 +    switch (violationCode) {
 +      case 1:
 +        return "data file size must be a non-negative integer";
 +      case 2:
 +        return "Invalid column name given.";
 +      case 3:
 +        return "Prev end row is greater than or equal to end row.";
 +      case 4:
 +        return "Invalid metadata row format";
 +      case 5:
 +        return "Row can not be less than " + MetadataTable.ID;
 +      case 6:
 +        return "Empty values are not allowed for any " + MetadataTable.NAME + " column";
 +      case 7:
 +        return "Lock not held in zookeeper by writer";
 +      case 8:
 +        return "Bulk load transaction no longer running";
 +    }
 +    return null;
 +  }
 +  
 +  @Override
 +  protected void finalize() {
 +    if (zooCache != null)
 +      zooCache.clear();
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 24b5605,0000000..7c4b8d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@@ -1,586 -1,0 +1,587 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.init;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.util.Arrays;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Locale;
 +import java.util.Map.Entry;
 +import java.util.UUID;
 +
 +import jline.console.ConsoleReader;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVWriter;
 +import org.apache.accumulo.core.iterators.user.VersioningIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.constraints.MetadataConstraints;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.util.TablePropUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.ZooDefs.Ids;
 +
 +import com.beust.jcommander.Parameter;
 +
 +/**
 + * This class is used to setup the directory structure and the root tablet to get an instance started
 + *
 + */
 +public class Initialize {
 +  private static final Logger log = Logger.getLogger(Initialize.class);
 +  private static final String DEFAULT_ROOT_USER = "root";
 +  public static final String TABLE_TABLETS_TABLET_DIR = "/table_info";
 +
 +  private static ConsoleReader reader = null;
 +  private static IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +  private static ConsoleReader getConsoleReader() throws IOException {
 +    if (reader == null)
 +      reader = new ConsoleReader();
 +    return reader;
 +  }
 +
 +  /**
 +   * Sets this class's ZooKeeper reader/writer.
 +   *
 +   * @param izoo
 +   *          reader/writer
 +   */
 +  static void setZooReaderWriter(IZooReaderWriter izoo) {
 +    zoo = izoo;
 +  }
 +
 +  /**
 +   * Gets this class's ZooKeeper reader/writer.
 +   *
 +   * @return reader/writer
 +   */
 +  static IZooReaderWriter getZooReaderWriter() {
 +    return zoo;
 +  }
 +
 +  private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
 +  static {
 +    initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K");
 +    initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
 +    initialMetadataConf.put(Property.TABLE_WALOG_ENABLED.getKey(), "true");
 +    initialMetadataConf.put(Property.TABLE_MAJC_RATIO.getKey(), "1");
 +    initialMetadataConf.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "64M");
 +    initialMetadataConf.put(Property.TABLE_CONSTRAINT_PREFIX.getKey() + "1", MetadataConstraints.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers", "10," + VersioningIterator.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers.opt.maxVersions", "1");
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers", "10," + VersioningIterator.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers.opt.maxVersions", "1");
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers", "10," + VersioningIterator.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers.opt.maxVersions", "1");
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.bulkLoadFilter", "20," + MetadataBulkLoadFilter.class.getName());
 +    initialMetadataConf.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
 +    initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
 +        String.format("%s,%s", TabletsSection.TabletColumnFamily.NAME, TabletsSection.CurrentLocationColumnFamily.NAME));
 +    initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s", TabletsSection.DataFileColumnFamily.NAME,
 +        TabletsSection.LogColumnFamily.NAME, TabletsSection.ServerColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME));
 +    initialMetadataConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
 +    initialMetadataConf.put(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
 +    initialMetadataConf.put(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
 +    initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
 +  }
 +
 +  static boolean checkInit(Configuration conf, VolumeManager fs, SiteConfiguration sconf) throws IOException {
 +    @SuppressWarnings("deprecation")
 +    String fsUri = sconf.get(Property.INSTANCE_DFS_URI);
 +    if (fsUri.equals(""))
 +      fsUri = FileSystem.getDefaultUri(conf).toString();
 +    log.info("Hadoop Filesystem is " + fsUri);
 +    log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
 +    log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
 +    log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
 +    if (!zookeeperAvailable()) {
 +      log.fatal("Zookeeper needs to be up and running in order to init. Exiting ...");
 +      return false;
 +    }
 +    if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) {
 +      ConsoleReader c = getConsoleReader();
 +      c.beep();
 +      c.println();
 +      c.println();
 +      c.println("Warning!!! Your instance secret is still set to the default, this is not secure. We highly recommend you change it.");
 +      c.println();
 +      c.println();
 +      c.println("You can change the instance secret in accumulo by using:");
 +      c.println("   bin/accumulo " + org.apache.accumulo.server.util.ChangeSecret.class.getName() + " oldPassword newPassword.");
 +      c.println("You will also need to edit your secret in your configuration file by adding the property instance.secret to your conf/accumulo-site.xml. Without this accumulo will not operate correctly");
 +    }
 +    try {
 +      if (isInitialized(fs)) {
 +        printInitializeFailureMessages(sconf);
 +        return false;
 +      }
 +    } catch (IOException e) {
 +      throw new IOException("Failed to check if filesystem already initialized", e);
 +    }
 +
 +    return true;
 +  }
 +
 +  static void printInitializeFailureMessages(SiteConfiguration sconf) {
 +    @SuppressWarnings("deprecation")
 +    Property INSTANCE_DFS_DIR = Property.INSTANCE_DFS_DIR;
 +    @SuppressWarnings("deprecation")
 +    Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI;
 +    String instanceDfsDir = sconf.get(INSTANCE_DFS_DIR);
 +    log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()))
 +        + " were previously initialized.");
 +    String instanceVolumes = sconf.get(Property.INSTANCE_VOLUMES);
 +    String instanceDfsUri = sconf.get(INSTANCE_DFS_URI);
 +
 +    if (!instanceVolumes.isEmpty()) {
 +      log.fatal("Change the property " + Property.INSTANCE_VOLUMES + " to use different filesystems,");
 +    } else if (!instanceDfsDir.isEmpty()) {
 +      log.fatal("Change the property " + INSTANCE_DFS_URI + " to use a different filesystem,");
 +    } else {
 +      log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_VOLUMES + " to use a different filesystem,");
 +    }
 +    log.fatal("or change the property " + INSTANCE_DFS_DIR + " to use a different directory.");
 +    log.fatal("The current value of " + INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|");
 +    log.fatal("The current value of " + INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|");
 +    log.fatal("The current value of " + Property.INSTANCE_VOLUMES + " is |" + instanceVolumes + "|");
 +  }
 +
 +  public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException {
 +    if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) {
 +      return false;
 +    }
 +
 +    // prompt user for instance name and root password early, in case they
 +    // abort, we don't leave an inconsistent HDFS/ZooKeeper structure
 +    String instanceNamePath;
 +    try {
 +      instanceNamePath = getInstanceNamePath(opts);
 +    } catch (Exception e) {
 +      log.fatal("Failed to talk to zookeeper", e);
 +      return false;
 +    }
 +    opts.rootpass = getRootPassword(opts);
 +    return initialize(opts, instanceNamePath, fs);
 +  }
 +
 +  public static boolean initialize(Opts opts, String instanceNamePath, VolumeManager fs) {
 +
 +    UUID uuid = UUID.randomUUID();
 +    // the actual disk locations of the root table and tablets
 +    String[] configuredVolumes = VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration());
 +    final String rootTabletDir = new Path(fs.choose(configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + RootTable.ID
 +        + RootTable.ROOT_TABLET_LOCATION).toString();
 +
 +    try {
 +      initZooKeeper(opts, uuid.toString(), instanceNamePath, rootTabletDir);
 +    } catch (Exception e) {
 +      log.fatal("Failed to initialize zookeeper", e);
 +      return false;
 +    }
 +
 +    try {
 +      initFileSystem(opts, fs, uuid, rootTabletDir);
 +    } catch (Exception e) {
 +      log.fatal("Failed to initialize filesystem", e);
 +
 +      if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES).trim().equals("")) {
 +        Configuration fsConf = CachedConfiguration.getInstance();
 +
 +        final String defaultFsUri = "file:///";
 +        String fsDefaultName = fsConf.get("fs.default.name", defaultFsUri), fsDefaultFS = fsConf.get("fs.defaultFS", defaultFsUri);
 +
 +        // Try to determine when we couldn't find an appropriate core-site.xml on the classpath
 +        if (defaultFsUri.equals(fsDefaultName) && defaultFsUri.equals(fsDefaultFS)) {
 +          log.fatal("Default filesystem value ('fs.defaultFS' or 'fs.default.name') of '" + defaultFsUri + "' was found in the Hadoop configuration");
 +          log.fatal("Please ensure that the Hadoop core-site.xml is on the classpath using 'general.classpaths' in accumulo-site.xml");
 +        }
 +      }
 +
 +      return false;
 +    }
 +
 +    try {
 +      initSecurity(opts, uuid.toString());
 +    } catch (Exception e) {
 +      log.fatal("Failed to initialize security", e);
 +      return false;
 +    }
 +    return true;
 +  }
 +
 +  private static boolean zookeeperAvailable() {
 +    try {
 +      return zoo.exists("/");
 +    } catch (KeeperException e) {
 +      return false;
 +    } catch (InterruptedException e) {
 +      return false;
 +    }
 +  }
 +
 +  private static void initDirs(VolumeManager fs, UUID uuid, String[] baseDirs, boolean print) throws IOException {
 +    for (String baseDir : baseDirs) {
 +      fs.mkdirs(new Path(new Path(baseDir, ServerConstants.VERSION_DIR), "" + ServerConstants.DATA_VERSION));
 +
 +      // create an instance id
 +      Path iidLocation = new Path(baseDir, ServerConstants.INSTANCE_ID_DIR);
 +      fs.mkdirs(iidLocation);
 +      fs.createNewFile(new Path(iidLocation, uuid.toString()));
 +      if (print)
 +        log.info("Initialized volume " + baseDir);
 +    }
 +  }
 +
 +  private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, String rootTabletDir) throws IOException {
 +    initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
 +
 +    // initialize initial metadata config in zookeeper
 +    initMetadataConfig();
 +
 +    String tableMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
 +        + TABLE_TABLETS_TABLET_DIR;
 +    String defaultMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
 +        + Constants.DEFAULT_TABLET_LOCATION;
 +
 +    // create table and default tablets directories
 +    createDirectories(fs, rootTabletDir, tableMetadataTabletDir, defaultMetadataTabletDir);
 +
 +    // populate the root tablet with info about the metadata tablets
 +    String fileName = rootTabletDir + Path.SEPARATOR + "00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
 +    createMetadataFile(fs, fileName, MetadataTable.ID, tableMetadataTabletDir, defaultMetadataTabletDir);
 +  }
 +
 +  /**
 +   * Create an rfile in the default tablet's directory for a new table. This method is used to create the initial root tablet contents, with information about
 +   * the metadata table's tablets
 +   *
 +   * @param volmanager
 +   *          The VolumeManager
 +   * @param fileName
 +   *          The location to create the file
 +   * @param tableId
 +   *          TableID that is being "created"
 +   * @param tableTabletDir
 +   *          The table_info directory for the new table
 +   * @param defaultTabletDir
 +   *          The default_tablet directory for the new table
 +   */
 +  private static void createMetadataFile(VolumeManager volmanager, String fileName, String tableId, String tableTabletDir, String defaultTabletDir)
 +      throws IOException {
 +    FileSystem fs = volmanager.getVolumeByPath(new Path(fileName)).getFileSystem();
 +    FileSKVWriter tabletWriter = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
 +    tabletWriter.startDefaultLocalityGroup();
 +
 +    Text splitPoint = TabletsSection.getRange().getEndKey().getRow();
 +    createEntriesForTablet(tabletWriter, tableId, tableTabletDir, null, splitPoint);
 +    createEntriesForTablet(tabletWriter, tableId, defaultTabletDir, splitPoint, null);
 +
 +    tabletWriter.close();
 +  }
 +
 +  private static void createEntriesForTablet(FileSKVWriter writer, String tableId, String tabletDir, Text tabletPrevEndRow, Text tabletEndRow)
 +      throws IOException {
 +    Text extent = new Text(KeyExtent.getMetadataEntry(new Text(tableId), tabletEndRow));
-     addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(Constants.UTF8)));
-     addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(Constants.UTF8)));
++    addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(UTF_8)));
++    addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8)));
 +    addEntry(writer, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tabletPrevEndRow));
 +  }
 +
 +  private static void addEntry(FileSKVWriter writer, Text row, ColumnFQ col, Value value) throws IOException {
 +    writer.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier(), 0), value);
 +  }
 +
 +  private static void createDirectories(VolumeManager fs, String... dirs) throws IOException {
 +    for (String s : dirs) {
 +      Path dir = new Path(s);
 +      try {
 +        FileStatus fstat = fs.getFileStatus(dir);
 +        // TODO Remove deprecation warning suppression when Hadoop1 support is dropped
 +        @SuppressWarnings("deprecation")
 +        boolean isDirectory = fstat.isDir();
 +        if (!isDirectory) {
 +          log.fatal("location " + dir + " exists but is not a directory");
 +          return;
 +        }
 +      } catch (FileNotFoundException fnfe) {
 +        // attempt to create directory, since it doesn't exist
 +        if (!fs.mkdirs(dir)) {
 +          log.fatal("unable to create directory " + dir);
 +          return;
 +        }
 +      }
 +    }
 +  }
 +
 +  private static void initZooKeeper(Opts opts, String uuid, String instanceNamePath, String rootTabletDir) throws KeeperException, InterruptedException {
 +    // setup basic data in zookeeper
 +    zoo.putPersistentData(Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
 +    zoo.putPersistentData(Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
 +
 +    // setup instance name
 +    if (opts.clearInstanceName)
 +      zoo.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
-     zoo.putPersistentData(instanceNamePath, uuid.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(instanceNamePath, uuid.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +    final byte[] EMPTY_BYTE_ARRAY = new byte[0], ZERO_CHAR_ARRAY = new byte[] {'0'};
 +
 +    // setup the instance
 +    String zkInstanceRoot = Constants.ZROOT + "/" + uuid;
 +    zoo.putPersistentData(zkInstanceRoot, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES, Constants.ZTABLES_INITIAL_ID, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES, new byte[0], NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewNamespaceState(uuid, Namespaces.DEFAULT_NAMESPACE_ID, Namespaces.DEFAULT_NAMESPACE, NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewNamespaceState(uuid, Namespaces.ACCUMULO_NAMESPACE_ID, Namespaces.ACCUMULO_NAMESPACE, NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewTableState(uuid, RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewTableState(uuid, MetadataTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, MetadataTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTSERVERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZPROBLEMS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_WALOGS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
-     zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
-     zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZNEXT_FILE, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZRECOVERY, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +  }
 +
 +  private static String getInstanceNamePath(Opts opts) throws IOException, KeeperException, InterruptedException {
 +    // setup the instance name
 +    String instanceName, instanceNamePath = null;
 +    boolean exists = true;
 +    do {
 +      if (opts.cliInstanceName == null) {
 +        instanceName = getConsoleReader().readLine("Instance name : ");
 +      } else {
 +        instanceName = opts.cliInstanceName;
 +      }
 +      if (instanceName == null)
 +        System.exit(0);
 +      instanceName = instanceName.trim();
 +      if (instanceName.length() == 0)
 +        continue;
 +      instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
 +      if (opts.clearInstanceName) {
 +        exists = false;
 +        break;
 +      } else if (exists = zoo.exists(instanceNamePath)) {
 +        String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : ");
 +        if (decision == null)
 +          System.exit(0);
 +        if (decision.length() == 1 && decision.toLowerCase(Locale.ENGLISH).charAt(0) == 'y') {
 +          opts.clearInstanceName = true;
 +          exists = false;
 +        }
 +      }
 +    } while (exists);
 +    return instanceNamePath;
 +  }
 +
 +  private static byte[] getRootPassword(Opts opts) throws IOException {
 +    if (opts.cliPassword != null) {
-       return opts.cliPassword.getBytes(Constants.UTF8);
++      return opts.cliPassword.getBytes(UTF_8);
 +    }
 +    String rootpass;
 +    String confirmpass;
 +    do {
 +      rootpass = getConsoleReader()
 +          .readLine("Enter initial password for " + DEFAULT_ROOT_USER + " (this may not be applicable for your security setup): ", '*');
 +      if (rootpass == null)
 +        System.exit(0);
 +      confirmpass = getConsoleReader().readLine("Confirm initial password for " + DEFAULT_ROOT_USER + ": ", '*');
 +      if (confirmpass == null)
 +        System.exit(0);
 +      if (!rootpass.equals(confirmpass))
 +        log.error("Passwords do not match");
 +    } while (!rootpass.equals(confirmpass));
-     return rootpass.getBytes(Constants.UTF8);
++    return rootpass.getBytes(UTF_8);
 +  }
 +
 +  private static void initSecurity(Opts opts, String iid) throws AccumuloSecurityException, ThriftSecurityException {
 +    AuditedSecurityOperation.getInstance(iid, true).initializeSecurity(SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()), DEFAULT_ROOT_USER,
 +        opts.rootpass);
 +  }
 +
 +  public static void initMetadataConfig() throws IOException {
 +    try {
 +      Configuration conf = CachedConfiguration.getInstance();
 +      int max = conf.getInt("dfs.replication.max", 512);
 +      // Hadoop 0.23 switched the min value configuration name
 +      int min = Math.max(conf.getInt("dfs.replication.min", 1), conf.getInt("dfs.namenode.replication.min", 1));
 +      if (max < 5)
 +        setMetadataReplication(max, "max");
 +      if (min > 5)
 +        setMetadataReplication(min, "min");
 +      for (Entry<String,String> entry : initialMetadataConf.entrySet()) {
 +        if (!TablePropUtil.setTableProperty(RootTable.ID, entry.getKey(), entry.getValue()))
 +          throw new IOException("Cannot create per-table property " + entry.getKey());
 +        if (!TablePropUtil.setTableProperty(MetadataTable.ID, entry.getKey(), entry.getValue()))
 +          throw new IOException("Cannot create per-table property " + entry.getKey());
 +      }
 +    } catch (Exception e) {
 +      log.fatal("error talking to zookeeper", e);
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  private static void setMetadataReplication(int replication, String reason) throws IOException {
 +    String rep = getConsoleReader().readLine(
 +        "Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME + " replication of 5. What do you want to set your "
 +            + MetadataTable.NAME + " replication to? (" + replication + ") ");
 +    if (rep == null || rep.length() == 0)
 +      rep = Integer.toString(replication);
 +    else
 +      // Lets make sure it's a number
 +      Integer.parseInt(rep);
 +    initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), rep);
 +  }
 +
 +  public static boolean isInitialized(VolumeManager fs) throws IOException {
 +    for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
 +      if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR)))
 +        return true;
 +    }
 +
 +    return false;
 +  }
 +
 +  private static void addVolumes(VolumeManager fs) throws IOException {
 +    HashSet<String> initializedDirs = new HashSet<String>();
 +    initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
 +
 +    HashSet<String> uinitializedDirs = new HashSet<String>();
 +    uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
 +    uinitializedDirs.removeAll(initializedDirs);
 +
 +    Path aBasePath = new Path(initializedDirs.iterator().next());
 +    Path iidPath = new Path(aBasePath, ServerConstants.INSTANCE_ID_DIR);
 +    Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
 +
 +    UUID uuid = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(iidPath, ServerConfiguration.getSiteConfiguration()));
 +
 +    if (ServerConstants.DATA_VERSION != Accumulo.getAccumuloPersistentVersion(versionPath.getFileSystem(CachedConfiguration.getInstance()), versionPath)) {
 +      throw new IOException("Accumulo " + Constants.VERSION + " cannot initialize data version " + Accumulo.getAccumuloPersistentVersion(fs));
 +    }
 +
 +    initDirs(fs, uuid, uinitializedDirs.toArray(new String[uinitializedDirs.size()]), true);
 +  }
 +
 +  static class Opts extends Help {
 +    @Parameter(names = "--add-volumes", description = "Initialize any uninitialized volumes listed in instance.volumes")
 +    boolean addVolumes = false;
 +    @Parameter(names = "--reset-security", description = "just update the security information")
 +    boolean resetSecurity = false;
 +    @Parameter(names = "--clear-instance-name", description = "delete any existing instance name without prompting")
 +    boolean clearInstanceName = false;
 +    @Parameter(names = "--instance-name", description = "the instance name, if not provided, will prompt")
 +    String cliInstanceName;
 +    @Parameter(names = "--password", description = "set the password on the command line")
 +    String cliPassword;
 +
 +    byte[] rootpass = null;
 +  }
 +
 +  public static void main(String[] args) {
 +    Opts opts = new Opts();
 +    opts.parseArgs(Initialize.class.getName(), args);
 +
 +    try {
 +      AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
 +      SecurityUtil.serverLogin(acuConf);
 +      Configuration conf = CachedConfiguration.getInstance();
 +
 +      VolumeManager fs = VolumeManagerImpl.get(acuConf);
 +
 +      if (opts.resetSecurity) {
 +        if (isInitialized(fs)) {
 +          opts.rootpass = getRootPassword(opts);
 +          initSecurity(opts, HdfsZooInstance.getInstance().getInstanceID());
 +        } else {
 +          log.fatal("Attempted to reset security on accumulo before it was initialized");
 +        }
 +      }
 +
 +      if (opts.addVolumes) {
 +        addVolumes(fs);
 +      }
 +
 +      if (!opts.resetSecurity && !opts.addVolumes)
 +        if (!doInit(opts, conf, fs))
 +          System.exit(-1);
 +    } catch (Exception e) {
 +      log.fatal(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +}


[14/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
index d0e6aea,0000000..258080c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
@@@ -1,775 -1,0 +1,777 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.nio.ByteBuffer;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.SecurityOperationsImpl;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.TColumn;
 +import org.apache.accumulo.core.data.thrift.TKeyExtent;
 +import org.apache.accumulo.core.data.thrift.TRange;
 +import org.apache.accumulo.core.master.thrift.FateOperation;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.security.handler.Authenticator;
 +import org.apache.accumulo.server.security.handler.Authorizor;
 +import org.apache.accumulo.server.security.handler.PermissionHandler;
 +import org.apache.accumulo.server.security.handler.ZKAuthenticator;
 +import org.apache.accumulo.server.security.handler.ZKAuthorizor;
 +import org.apache.accumulo.server.security.handler.ZKPermHandler;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * Utility class for performing various security operations with the appropriate checks
 + */
 +public class SecurityOperation {
 +  private static final Logger log = Logger.getLogger(SecurityOperationsImpl.class);
 +
 +  protected Authorizor authorizor;
 +  protected Authenticator authenticator;
 +  protected PermissionHandler permHandle;
 +  private static String rootUserName = null;
 +  private final ZooCache zooCache;
 +  private final String ZKUserPath;
 +
 +  static SecurityOperation instance;
 +
 +  public static synchronized SecurityOperation getInstance() {
 +    String instanceId = HdfsZooInstance.getInstance().getInstanceID();
 +    return getInstance(instanceId, false);
 +  }
 +
 +  public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) {
 +    if (instance == null) {
 +      instance = new SecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, initialize),
 +          instanceId);
 +    }
 +    return instance;
 +  }
 +
 +  protected static Authorizor getAuthorizor(String instanceId, boolean initialize) {
 +    Authorizor toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHORIZOR, Authorizor.class,
 +        ZKAuthorizor.getInstance());
 +    toRet.initialize(instanceId, initialize);
 +    return toRet;
 +  }
 +
 +  protected static Authenticator getAuthenticator(String instanceId, boolean initialize) {
 +    Authenticator toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHENTICATOR, Authenticator.class,
 +        ZKAuthenticator.getInstance());
 +    toRet.initialize(instanceId, initialize);
 +    return toRet;
 +  }
 +
 +  protected static PermissionHandler getPermHandler(String instanceId, boolean initialize) {
 +    PermissionHandler toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_PERMISSION_HANDLER,
 +        PermissionHandler.class, ZKPermHandler.getInstance());
 +    toRet.initialize(instanceId, initialize);
 +    return toRet;
 +  }
 +
 +  protected SecurityOperation(String instanceId) {
 +    ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
 +    zooCache = new ZooCache();
 +  }
 +
 +  public SecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) {
 +    this(instanceId);
 +    authorizor = author;
 +    authenticator = authent;
 +    permHandle = pm;
 +
 +    if (!authorizor.validSecurityHandlers(authenticator, pm) || !authenticator.validSecurityHandlers(authorizor, pm)
 +        || !permHandle.validSecurityHandlers(authent, author))
 +      throw new RuntimeException(authorizor + ", " + authenticator + ", and " + pm
 +          + " do not play nice with eachother. Please choose authentication and authorization mechanisms that are compatible with one another.");
 +  }
 +
 +  public void initializeSecurity(TCredentials credentials, String rootPrincipal, byte[] token) throws AccumuloSecurityException, ThriftSecurityException {
 +    authenticate(credentials);
 +
 +    if (!isSystemUser(credentials))
 +      throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    authenticator.initializeSecurity(credentials, rootPrincipal, token);
 +    authorizor.initializeSecurity(credentials, rootPrincipal);
 +    permHandle.initializeSecurity(credentials, rootPrincipal);
 +    try {
 +      permHandle.grantTablePermission(rootPrincipal, MetadataTable.ID, TablePermission.ALTER_TABLE);
 +    } catch (TableNotFoundException e) {
 +      // Shouldn't happen
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  public synchronized String getRootUsername() {
 +    if (rootUserName == null)
-       rootUserName = new String(zooCache.get(ZKUserPath), Constants.UTF8);
++      rootUserName = new String(zooCache.get(ZKUserPath), UTF_8);
 +    return rootUserName;
 +  }
 +
 +  public boolean isSystemUser(TCredentials credentials) {
 +    return SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName());
 +  }
 +
 +  protected void authenticate(TCredentials credentials) throws ThriftSecurityException {
 +    if (!credentials.getInstanceId().equals(HdfsZooInstance.getInstance().getInstanceID()))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.INVALID_INSTANCEID);
 +
 +    Credentials creds = Credentials.fromThrift(credentials);
 +    if (isSystemUser(credentials)) {
 +      if (!(SystemCredentials.get().equals(creds))) {
 +        throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
 +      }
 +    } else {
 +      try {
 +        if (!authenticator.authenticateUser(creds.getPrincipal(), creds.getToken())) {
 +          throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
 +        }
 +      } catch (AccumuloSecurityException e) {
 +        log.debug(e);
 +        throw e.asThriftException();
 +      }
 +    }
 +  }
 +
 +  public boolean canAskAboutUser(TCredentials credentials, String user) throws ThriftSecurityException {
 +    // Authentication done in canPerformSystemActions
 +    if (!(canPerformSystemActions(credentials) || credentials.getPrincipal().equals(user)))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return true;
 +  }
 +
 +  public boolean authenticateUser(TCredentials credentials, TCredentials toAuth) throws ThriftSecurityException {
 +    canAskAboutUser(credentials, toAuth.getPrincipal());
 +    // User is already authenticated from canAskAboutUser
 +    if (credentials.equals(toAuth))
 +      return true;
 +    try {
 +      Credentials toCreds = Credentials.fromThrift(toAuth);
 +      return authenticator.authenticateUser(toCreds.getPrincipal(), toCreds.getToken());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public Authorizations getUserAuthorizations(TCredentials credentials, String user) throws ThriftSecurityException {
 +    authenticate(credentials);
 +
 +    targetUserExists(user);
 +
 +    if (!credentials.getPrincipal().equals(user) && !hasSystemPermission(credentials, SystemPermission.SYSTEM, false))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    try {
 +      return authorizor.getCachedUserAuthorizations(user);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public Authorizations getUserAuthorizations(TCredentials credentials) throws ThriftSecurityException {
 +    // system user doesn't need record-level authorizations for the tables it reads
 +    if (isSystemUser(credentials)) {
 +      authenticate(credentials);
 +      return Authorizations.EMPTY;
 +    }
 +    return getUserAuthorizations(credentials, credentials.getPrincipal());
 +  }
 +
 +  public boolean userHasAuthorizations(TCredentials credentials, List<ByteBuffer> list) throws ThriftSecurityException {
 +    authenticate(credentials);
 +
 +    if (isSystemUser(credentials)) {
 +      // system user doesn't need record-level authorizations for the tables it reads (for now)
 +      return list.isEmpty();
 +    }
 +
 +    try {
 +      return authorizor.isValidAuthorizations(credentials.getPrincipal(), list);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  private boolean hasSystemPermission(TCredentials credentials, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
 +    return hasSystemPermissionWithNamespaceId(credentials, permission, null, useCached);
 +  }
 +
 +  /**
 +   * Checks if a user has a system permission
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  private boolean hasSystemPermissionWithNamespaceId(TCredentials credentials, SystemPermission permission, String namespaceId, boolean useCached)
 +      throws ThriftSecurityException {
 +    if (isSystemUser(credentials))
 +      return true;
 +
 +    if (_hasSystemPermission(credentials.getPrincipal(), permission, useCached))
 +      return true;
 +    if (namespaceId != null) {
 +      return _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
 +    }
 +
 +    return false;
 +  }
 +
 +  /**
 +   * Checks if a user has a system permission<br/>
 +   * This cannot check if a system user has permission.
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  private boolean _hasSystemPermission(String user, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
 +    if (user.equals(getRootUsername()))
 +      return true;
 +
 +    targetUserExists(user);
 +
 +    try {
 +      if (useCached)
 +        return permHandle.hasCachedSystemPermission(user, permission);
 +      return permHandle.hasSystemPermission(user, permission);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  /**
 +   * Checks if a user has a table permission
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  protected boolean hasTablePermission(TCredentials credentials, String tableId, String namespaceId, TablePermission permission, boolean useCached) throws ThriftSecurityException {
 +    if (isSystemUser(credentials))
 +      return true;
 +    return _hasTablePermission(credentials.getPrincipal(), tableId, permission, useCached)
 +        || _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
 +  }
 +
 +  /**
 +   * Checks if a user has a table permission<br/>
 +   * This cannot check if a system user has permission.
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  protected boolean _hasTablePermission(String user, String table, TablePermission permission, boolean useCached) throws ThriftSecurityException {
 +    targetUserExists(user);
 +
 +    if ((table.equals(MetadataTable.ID) || table.equals(RootTable.ID)) && permission.equals(TablePermission.READ))
 +      return true;
 +
 +    try {
 +      if (useCached)
 +        return permHandle.hasCachedTablePermission(user, table, permission);
 +      return permHandle.hasTablePermission(user, table, permission);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(user, SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  /**
 +   * Checks if a user has a namespace permission<br/>
 +   * This cannot check if a system user has permission.
 +   * 
 +   * @return true if a user exists and has permission; false otherwise
 +   */
 +  protected boolean _hasNamespacePermission(String user, String namespace, NamespacePermission permission, boolean useCached) throws ThriftSecurityException {
 +    if (permission == null)
 +      return false;
 +
 +    targetUserExists(user);
 +
 +    if (namespace.equals(Namespaces.ACCUMULO_NAMESPACE_ID) && permission.equals(NamespacePermission.READ))
 +      return true;
 +
 +    try {
 +      if (useCached)
 +        return permHandle.hasCachedNamespacePermission(user, namespace, permission);
 +      return permHandle.hasNamespacePermission(user, namespace, permission);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(user, SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  // some people just aren't allowed to ask about other users; here are those who can ask
 +  private boolean canAskAboutOtherUsers(TCredentials credentials, String user) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return credentials.getPrincipal().equals(user) || hasSystemPermission(credentials, SystemPermission.SYSTEM, false)
 +        || hasSystemPermission(credentials, SystemPermission.CREATE_USER, false) || hasSystemPermission(credentials, SystemPermission.ALTER_USER, false)
 +        || hasSystemPermission(credentials, SystemPermission.DROP_USER, false);
 +  }
 +
 +  private void targetUserExists(String user) throws ThriftSecurityException {
 +    if (user.equals(getRootUsername()))
 +      return;
 +    try {
 +      if (!authenticator.userExists(user))
 +        throw new ThriftSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST);
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public boolean canScan(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, true);
 +  }
 +
 +  public boolean canScan(TCredentials credentials, String tableId, String namespaceId, TRange range, List<TColumn> columns, List<IterInfo> ssiList,
 +      Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
 +    return canScan(credentials, tableId, namespaceId);
 +  }
 +
 +  public boolean canScan(TCredentials credentials, String table, String namespaceId, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList,
 +      Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
 +    return canScan(credentials, table, namespaceId);
 +  }
 +
 +  public boolean canWrite(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasTablePermission(credentials, tableId, namespaceId, TablePermission.WRITE, true);
 +  }
 +
 +  public boolean canConditionallyUpdate(TCredentials credentials, String tableID, String namespaceId, List<ByteBuffer> authorizations) throws ThriftSecurityException {
 +
 +    authenticate(credentials);
 +
 +    return hasTablePermission(credentials, tableID, namespaceId, TablePermission.WRITE, true) && hasTablePermission(credentials, tableID, namespaceId, TablePermission.READ, true);
 +  }
 +
 +  public boolean canSplitTablet(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasSystemPermissionWithNamespaceId(credentials, SystemPermission.SYSTEM, namespaceId, false)
 +        || hasTablePermission(credentials, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  /**
 +   * This is the check to perform any system action. This includes tserver's loading of a tablet, shutting the system down, or altering system properties.
 +   */
 +  public boolean canPerformSystemActions(TCredentials credentials) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermission(credentials, SystemPermission.SYSTEM, false);
 +  }
 +
 +  public boolean canFlush(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canAlterTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false)
 +        || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false);
 +  }
 +
 +  public boolean canCreateTable(TCredentials c, String table, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, namespaceId, false);
 +  }
 +
 +  public boolean canRenameTable(TCredentials c, String tableId, String oldTableName, String newTableName, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canCloneTable(TCredentials c, String tableId, String tableName, String destinationNamespaceId, String srcNamespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, destinationNamespaceId, false) && hasTablePermission(c, tableId, srcNamespaceId, TablePermission.READ, false);
 +  }
 +
 +  public boolean canDeleteTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.DROP_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.DROP_TABLE, false);
 +  }
 +
 +  public boolean canOnlineOfflineTable(TCredentials c, String tableId, FateOperation op, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
 +        || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canMerge(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
 +        || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
 +  }
 +
 +  public boolean canDeleteRange(TCredentials c, String tableId, String tableName, Text startRow, Text endRow, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
 +  }
 +
 +  public boolean canBulkImport(TCredentials c, String tableId, String tableName, String dir, String failDir, String namespaceId) throws ThriftSecurityException {
 +    return canBulkImport(c, tableId, namespaceId);
 +  }
 +
 +  public boolean canBulkImport(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasTablePermission(c, tableId, namespaceId, TablePermission.BULK_IMPORT, false);
 +  }
 +
 +  public boolean canCompact(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
 +        || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
 +  }
 +
 +  public boolean canChangeAuthorizations(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermission(c, SystemPermission.ALTER_USER, false);
 +  }
 +
 +  public boolean canChangePassword(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    return c.getPrincipal().equals(user) || hasSystemPermission(c, SystemPermission.ALTER_USER, false);
 +  }
 +
 +  public boolean canCreateUser(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermission(c, SystemPermission.CREATE_USER, false);
 +  }
 +
 +  public boolean canDropUser(TCredentials c, String user) throws ThriftSecurityException {
 +    authenticate(c);
 +    if (user.equals(getRootUsername()))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return hasSystemPermission(c, SystemPermission.DROP_USER, false);
 +  }
 +
 +  public boolean canGrantSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
 +    authenticate(c);
 +    // can't grant GRANT
 +    if (sysPerm.equals(SystemPermission.GRANT))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
 +    return hasSystemPermission(c, SystemPermission.GRANT, false);
 +  }
 +
 +  public boolean canGrantTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
 +  }
 +
 +  public boolean canGrantNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
 +    return canModifyNamespacePermission(c, user, namespace);
 +  }
 +
 +  private boolean canModifyNamespacePermission(TCredentials c, String user, String namespace) throws ThriftSecurityException {
 +    authenticate(c);
 +    // The one case where Table/SystemPermission -> NamespacePermission breaks down. The alternative is to make SystemPermission.ALTER_NAMESPACE provide
 +    // NamespacePermission.GRANT & ALTER_NAMESPACE, but then it would cause some permission checks to succeed with GRANT when they shouldn't
 +    
 +    // This is a bit hackier then I (vines) wanted, but I think this one hackiness makes the overall SecurityOperations more succinct.
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_NAMESPACE, namespace, false)
 +        || hasNamespacePermission(c, c.principal, namespace, NamespacePermission.GRANT);
 +  }
 +
 +  public boolean canRevokeSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
 +    authenticate(c);
 +    // can't modify root user
 +    if (user.equals(getRootUsername()))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    // can't revoke GRANT
 +    if (sysPerm.equals(SystemPermission.GRANT))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
 +
 +    return hasSystemPermission(c, SystemPermission.GRANT, false);
 +  }
 +
 +  public boolean canRevokeTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
 +    authenticate(c);
 +    return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
 +  }
 +
 +  public boolean canRevokeNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
 +    return canModifyNamespacePermission(c, user, namespace);
 +  }
 +
 +  public void changeAuthorizations(TCredentials credentials, String user, Authorizations authorizations) throws ThriftSecurityException {
 +    if (!canChangeAuthorizations(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      authorizor.changeAuthorizations(user, authorizations);
 +      log.info("Changed authorizations for user " + user + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException ase) {
 +      throw ase.asThriftException();
 +    }
 +  }
 +
 +  public void changePassword(TCredentials credentials, Credentials toChange) throws ThriftSecurityException {
 +    if (!canChangePassword(credentials, toChange.getPrincipal()))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      AuthenticationToken token = toChange.getToken();
 +      authenticator.changePassword(toChange.getPrincipal(), token);
 +      log.info("Changed password for user " + toChange.getPrincipal() + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void createUser(TCredentials credentials, Credentials newUser, Authorizations authorizations) throws ThriftSecurityException {
 +    if (!canCreateUser(credentials, newUser.getPrincipal()))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      AuthenticationToken token = newUser.getToken();
 +      authenticator.createUser(newUser.getPrincipal(), token);
 +      authorizor.initUser(newUser.getPrincipal());
 +      permHandle.initUser(newUser.getPrincipal());
 +      log.info("Created user " + newUser.getPrincipal() + " at the request of user " + credentials.getPrincipal());
 +      if (canChangeAuthorizations(credentials, newUser.getPrincipal()))
 +        authorizor.changeAuthorizations(newUser.getPrincipal(), authorizations);
 +    } catch (AccumuloSecurityException ase) {
 +      throw ase.asThriftException();
 +    }
 +  }
 +
 +  public void dropUser(TCredentials credentials, String user) throws ThriftSecurityException {
 +    if (!canDropUser(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      authorizor.dropUser(user);
 +      authenticator.dropUser(user);
 +      permHandle.cleanUser(user);
 +      log.info("Deleted user " + user + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void grantSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
 +    if (!canGrantSystem(credentials, user, permissionById))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.grantSystemPermission(user, permissionById);
 +      log.info("Granted system permission " + permissionById + " for user " + user + " at the request of user " + credentials.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void grantTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
 +    if (!canGrantTable(c, user, tableId, namespaceId))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.grantTablePermission(user, tableId, permission);
 +      log.info("Granted table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void grantNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
 +    if (!canGrantNamespace(c, user, namespace))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.grantNamespacePermission(user, namespace, permission);
 +      log.info("Granted namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
 +          + c.getPrincipal());
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void revokeSystemPermission(TCredentials credentials, String user, SystemPermission permission) throws ThriftSecurityException {
 +    if (!canRevokeSystem(credentials, user, permission))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.revokeSystemPermission(user, permission);
 +      log.info("Revoked system permission " + permission + " for user " + user + " at the request of user " + credentials.getPrincipal());
 +
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void revokeTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
 +    if (!canRevokeTable(c, user, tableId, namespaceId))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.revokeTablePermission(user, tableId, permission);
 +      log.info("Revoked table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
 +
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void revokeNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
 +    if (!canRevokeNamespace(c, user, namespace))
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +    targetUserExists(user);
 +
 +    try {
 +      permHandle.revokeNamespacePermission(user, namespace, permission);
 +      log.info("Revoked namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
 +          + c.getPrincipal());
 +
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public boolean hasSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
 +    if (!canAskAboutOtherUsers(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return _hasSystemPermission(user, permissionById, false);
 +  }
 +
 +  public boolean hasTablePermission(TCredentials credentials, String user, String tableId, TablePermission permissionById) throws ThriftSecurityException {
 +    if (!canAskAboutOtherUsers(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return _hasTablePermission(user, tableId, permissionById, false);
 +  }
 +
 +  public boolean hasNamespacePermission(TCredentials credentials, String user, String namespace, NamespacePermission permissionById)
 +      throws ThriftSecurityException {
 +    if (!canAskAboutOtherUsers(credentials, user))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    return _hasNamespacePermission(user, namespace, permissionById, false);
 +  }
 +
 +  public Set<String> listUsers(TCredentials credentials) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    try {
 +      return authenticator.listUsers();
 +    } catch (AccumuloSecurityException e) {
 +      throw e.asThriftException();
 +    }
 +  }
 +
 +  public void deleteTable(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
 +    if (!canDeleteTable(credentials, tableId, namespaceId))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      permHandle.cleanTablePermissions(tableId);
 +    } catch (AccumuloSecurityException e) {
 +      e.setUser(credentials.getPrincipal());
 +      throw e.asThriftException();
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public void deleteNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
 +    if (!canDeleteNamespace(credentials, namespace))
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +    try {
 +      permHandle.cleanNamespacePermissions(namespace);
 +    } catch (AccumuloSecurityException e) {
 +      e.setUser(credentials.getPrincipal());
 +      throw e.asThriftException();
 +    } catch (NamespaceNotFoundException e) {
 +      throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
 +    }
 +  }
 +
 +  public boolean canExport(TCredentials credentials, String tableId, String tableName, String exportDir, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, false);
 +  }
 +
 +  public boolean canImport(TCredentials credentials, String tableName, String importDir, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.CREATE_TABLE, namespaceId, false);
 +  }
 +
 +  public boolean canAlterNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
 +  }
 +
 +  public boolean canCreateNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
 +    return canCreateNamespace(credentials);
 +  }
 +
 +  private boolean canCreateNamespace(TCredentials credentials) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermission(credentials, SystemPermission.CREATE_NAMESPACE, false);
 +  }
 +
 +  public boolean canDeleteNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.DROP_NAMESPACE, namespaceId, false);
 +  }
 +
 +  public boolean canRenameNamespace(TCredentials credentials, String namespaceId, String oldName, String newName) throws ThriftSecurityException {
 +    authenticate(credentials);
 +    return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 19e7ff7,0000000..872fe11
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@@ -1,149 -1,0 +1,151 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.security.MessageDigest;
 +import java.security.NoSuchAlgorithmException;
 +import java.security.SecurityPermission;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.hadoop.io.Writable;
 +
 +/**
 + * Credentials for the system services.
 + * 
 + * @since 1.6.0
 + */
 +public final class SystemCredentials extends Credentials {
 +
 +  private static final SecurityPermission SYSTEM_CREDENTIALS_PERMISSION = new SecurityPermission("systemCredentialsPermission");
 +
 +  private static SystemCredentials SYSTEM_CREDS = null;
 +  private static final String SYSTEM_PRINCIPAL = "!SYSTEM";
 +
 +  private final TCredentials AS_THRIFT;
 +
 +  SystemCredentials(Instance instance) {
 +    super(SYSTEM_PRINCIPAL, SystemToken.get(instance));
 +    AS_THRIFT = super.toThrift(instance);
 +  }
 +
 +  public static SystemCredentials get() {
 +    check_permission();
 +    if (SYSTEM_CREDS == null) {
 +      SYSTEM_CREDS = new SystemCredentials(HdfsZooInstance.getInstance());
 +    }
 +    return SYSTEM_CREDS;
 +  }
 +
 +  private static void check_permission() {
 +    SecurityManager sm = System.getSecurityManager();
 +    if (sm != null) {
 +      sm.checkPermission(SYSTEM_CREDENTIALS_PERMISSION);
 +    }
 +  }
 +
 +  public static SystemCredentials get(Instance instance) {
 +    check_permission();
 +    /* Special case to avoid duplicating SYSTEM_CREDS */
 +    if (null != SYSTEM_CREDS) {
 +      if (SYSTEM_CREDS.AS_THRIFT.getInstanceId().equals(instance.getInstanceID())) {
 +        return SYSTEM_CREDS;
 +      }
 +    }
 +    return new SystemCredentials(instance);
 +  }
 +
 +  @Override
 +  public TCredentials toThrift(Instance instance) {
 +    if (!AS_THRIFT.getInstanceId().equals(instance.getInstanceID()))
 +      throw new IllegalArgumentException("Unexpected instance used for " + SystemCredentials.class.getSimpleName() + ": " + instance.getInstanceID());
 +    return AS_THRIFT;
 +  }
 +
 +  /**
 +   * An {@link AuthenticationToken} type for Accumulo servers for inter-server communication.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static final class SystemToken extends PasswordToken {
 +
 +    /**
 +     * A Constructor for {@link Writable}.
 +     */
 +    public SystemToken() {}
 +
 +    private SystemToken(byte[] systemPassword) {
 +      super(systemPassword);
 +    }
 +
 +    private static SystemToken get(Instance instance) {
-       byte[] instanceIdBytes = instance.getInstanceID().getBytes(Constants.UTF8);
++      byte[] instanceIdBytes = instance.getInstanceID().getBytes(UTF_8);
 +      byte[] confChecksum;
 +      MessageDigest md;
 +      try {
 +        md = MessageDigest.getInstance(Constants.PW_HASH_ALGORITHM);
 +      } catch (NoSuchAlgorithmException e) {
 +        throw new RuntimeException("Failed to compute configuration checksum", e);
 +      }
 +
 +      // seed the config with the version and instance id, so at least it's not empty
-       md.update(ServerConstants.WIRE_VERSION.toString().getBytes(Constants.UTF8));
++      md.update(ServerConstants.WIRE_VERSION.toString().getBytes(UTF_8));
 +      md.update(instanceIdBytes);
 +
 +      for (Entry<String,String> entry : ServerConfiguration.getSiteConfiguration()) {
 +        // only include instance properties
 +        if (entry.getKey().startsWith(Property.INSTANCE_PREFIX.toString())) {
-           md.update(entry.getKey().getBytes(Constants.UTF8));
-           md.update(entry.getValue().getBytes(Constants.UTF8));
++          md.update(entry.getKey().getBytes(UTF_8));
++          md.update(entry.getValue().getBytes(UTF_8));
 +        }
 +      }
 +      confChecksum = md.digest();
 +
 +      int wireVersion = ServerConstants.WIRE_VERSION;
 +
 +      ByteArrayOutputStream bytes = new ByteArrayOutputStream(3 * (Integer.SIZE / Byte.SIZE) + instanceIdBytes.length + confChecksum.length);
 +      DataOutputStream out = new DataOutputStream(bytes);
 +      try {
 +        out.write(wireVersion * -1);
 +        out.write(instanceIdBytes.length);
 +        out.write(instanceIdBytes);
 +        out.write(confChecksum.length);
 +        out.write(confChecksum);
 +      } catch (IOException e) {
 +        // this is impossible with ByteArrayOutputStream; crash hard if this happens
 +        throw new RuntimeException(e);
 +      }
 +      return new SystemToken(Base64.encodeBase64(bytes.toByteArray()));
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
index 5f6ff71,0000000..fa1632c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
@@@ -1,214 -1,0 +1,216 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +// Utility class for adding all authentication info into ZK
 +public final class ZKAuthenticator implements Authenticator {
 +  static final Logger log = Logger.getLogger(ZKAuthenticator.class);
 +  private static Authenticator zkAuthenticatorInstance = null;
 +
 +  private String ZKUserPath;
 +  private final ZooCache zooCache;
 +
 +  public static synchronized Authenticator getInstance() {
 +    if (zkAuthenticatorInstance == null)
 +      zkAuthenticatorInstance = new ZKAuthenticator();
 +    return zkAuthenticatorInstance;
 +  }
 +
 +  public ZKAuthenticator() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException {
 +    try {
 +      // remove old settings from zookeeper first, if any
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        if (zoo.exists(ZKUserPath)) {
 +          zoo.recursiveDelete(ZKUserPath, NodeMissingPolicy.SKIP);
 +          log.info("Removed " + ZKUserPath + "/" + " from zookeeper");
 +        }
 +
 +        // prep parent node of users with root username
-         zoo.putPersistentData(ZKUserPath, principal.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++        zoo.putPersistentData(ZKUserPath, principal.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +        constructUser(principal, ZKSecurityTool.createPass(token));
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (AccumuloException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets up the user in ZK for the provided user. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void constructUser(String user, byte[] pass) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      zoo.putPrivatePersistentData(ZKUserPath + "/" + user, pass, NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  @Override
 +  public Set<String> listUsers() {
 +    return new TreeSet<String>(zooCache.getChildren(ZKUserPath));
 +  }
 +
 +  /**
 +   * Creates a user with no permissions whatsoever
 +   */
 +  @Override
 +  public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
 +    try {
 +      if (!(token instanceof PasswordToken))
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
 +      PasswordToken pt = (PasswordToken) token;
 +      constructUser(principal, ZKSecurityTool.createPass(pt.getPassword()));
 +    } catch (KeeperException e) {
 +      if (e.code().equals(KeeperException.Code.NODEEXISTS))
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_EXISTS, e);
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (AccumuloException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
 +    }
 +  }
 +
 +  @Override
 +  public void dropUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        ZooReaderWriter.getInstance().recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    }
 +  }
 +
 +  @Override
 +  public void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException {
 +    if (!(token instanceof PasswordToken))
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
 +    PasswordToken pt = (PasswordToken) token;
 +    if (userExists(principal)) {
 +      try {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + principal);
 +          ZooReaderWriter.getInstance().putPrivatePersistentData(ZKUserPath + "/" + principal, ZKSecurityTool.createPass(pt.getPassword()),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      } catch (KeeperException e) {
 +        log.error(e, e);
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
 +      } catch (InterruptedException e) {
 +        log.error(e, e);
 +        throw new RuntimeException(e);
 +      } catch (AccumuloException e) {
 +        log.error(e, e);
 +        throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
 +      }
 +    } else
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
 +  }
 +
 +  /**
 +   * Checks if a user exists
 +   */
 +  @Override
 +  public boolean userExists(String user) {
 +    return zooCache.get(ZKUserPath + "/" + user) != null;
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm) {
 +    return true;
 +  }
 +
 +  @Override
 +  public boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
 +    if (!(token instanceof PasswordToken))
 +      throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
 +    PasswordToken pt = (PasswordToken) token;
 +    byte[] pass;
 +    String zpath = ZKUserPath + "/" + principal;
 +    pass = zooCache.get(zpath);
 +    boolean result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
 +    if (!result) {
 +      zooCache.clear(zpath);
 +      pass = zooCache.get(zpath);
 +      result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
 +    }
 +    return result;
 +  }
 +
 +  @Override
 +  public Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes() {
 +    Set<Class<? extends AuthenticationToken>> cs = new HashSet<Class<? extends AuthenticationToken>>();
 +    cs.add(PasswordToken.class);
 +    return cs;
 +  }
 +
 +  @Override
 +  public boolean validTokenClass(String tokenClass) {
 +    return tokenClass.equals(PasswordToken.class.getName());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 75b73fc,0000000..78597ad
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,171 -1,0 +1,172 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.nio.ByteBuffer;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class ZKAuthorizor implements Authorizor {
 +  private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
 +  private static Authorizor zkAuthorizorInstance = null;
 +
 +  private final String ZKUserAuths = "/Authorizations";
 +
 +  private String ZKUserPath;
 +  private final ZooCache zooCache;
 +
 +  public static synchronized Authorizor getInstance() {
 +    if (zkAuthorizorInstance == null)
 +      zkAuthorizorInstance = new ZKAuthorizor();
 +    return zkAuthorizorInstance;
 +  }
 +
 +  public ZKAuthorizor() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
 +  }
 +
 +  @Override
 +  public Authorizations getCachedUserAuthorizations(String user) {
 +    byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
 +    if (authsBytes != null)
 +      return ZKSecurityTool.convertAuthorizations(authsBytes);
 +    return Authorizations.EMPTY;
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
 +    return true;
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    // create the root user with all system privileges, no table privileges, and no record-level authorizations
 +    Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
 +    for (SystemPermission p : SystemPermission.values())
 +      rootPerms.add(p);
 +    Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
 +    // Allow the root user to flush the metadata tables
 +    tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +
 +    try {
 +      // prep parent node of users with root username
 +      if (!zoo.exists(ZKUserPath))
-         zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++        zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +      initUser(rootuser);
 +      zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initUser(String user) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void dropUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
 +        zooCache.clear(ZKUserPath + "/" + user);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +
 +    }
 +  }
 +
 +  @Override
 +  public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
 +            NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
 +    Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
 +    for (ByteBuffer auth : auths)
 +      if (!userauths.contains(auth))
 +        return false;
 +    return true;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index be4806b,0000000..41f880d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,515 -1,0 +1,516 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.security.handler;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.Code;
 +
 +/**
 + *
 + */
 +public class ZKPermHandler implements PermissionHandler {
 +  private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
 +  private static PermissionHandler zkPermHandlerInstance = null;
 +
 +  private String ZKUserPath;
 +  private String ZKTablePath;
 +  private String ZKNamespacePath;
 +  private final ZooCache zooCache;
 +  private final String ZKUserSysPerms = "/System";
 +  private final String ZKUserTablePerms = "/Tables";
 +  private final String ZKUserNamespacePerms = "/Namespaces";
 +
 +  public static synchronized PermissionHandler getInstance() {
 +    if (zkPermHandlerInstance == null)
 +      zkPermHandlerInstance = new ZKPermHandler();
 +    return zkPermHandlerInstance;
 +  }
 +
 +  @Override
 +  public void initialize(String instanceId, boolean initialize) {
 +    ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
 +    ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
 +    ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
 +  }
 +
 +  public ZKPermHandler() {
 +    zooCache = new ZooCache();
 +  }
 +
 +  @Override
 +  public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
 +    byte[] serializedPerms;
 +    final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
 +      zrw.sync(path);
 +      serializedPerms = zrw.getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        // maybe the table was just deleted?
 +        try {
 +          // check for existence:
 +          zrw.getData(ZKTablePath + "/" + table, null);
 +          // it's there, you don't have permission
 +          return false;
 +        } catch (InterruptedException ex) {
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +          return false;
 +        } catch (KeeperException ex) {
 +          // not there, throw an informative exception
 +          if (e.code() == Code.NONODE) {
 +            throw new TableNotFoundException(null, table, "while checking permissions");
 +          }
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +        }
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
 +    byte[] serializedPerms;
 +    final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
 +      zrw.sync(path);
 +      serializedPerms = zrw.getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        // maybe the namespace was just deleted?
 +        try {
 +          // check for existence:
 +          zrw.getData(ZKNamespacePath + "/" + namespace, null);
 +          // it's there, you don't have permission
 +          return false;
 +        } catch (InterruptedException ex) {
 +          log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
 +          return false;
 +        } catch (KeeperException ex) {
 +          // not there, throw an informative exception
 +          if (e.code() == Code.NONODE) {
 +            throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
 +          }
 +          log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +        }
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
 +      NamespaceNotFoundException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +    if (serializedPerms != null) {
 +      return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
 +    }
 +    return false;
 +  }
 +
 +  @Override
 +  public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    try {
 +      byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +      Set<SystemPermission> perms;
 +      if (permBytes == null) {
 +        perms = new TreeSet<SystemPermission>();
 +      } else {
 +        perms = ZKSecurityTool.convertSystemPermissions(permBytes);
 +      }
 +
 +      if (perms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear();
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
 +    Set<TablePermission> tablePerms;
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +    if (serializedPerms != null)
 +      tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
 +    else
 +      tablePerms = new TreeSet<TablePermission>();
 +
 +    try {
 +      if (tablePerms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
 +              ZKSecurityTool.convertTablePermissions(tablePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
 +    Set<NamespacePermission> namespacePerms;
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +    if (serializedPerms != null)
 +      namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
 +    else
 +      namespacePerms = new TreeSet<NamespacePermission>();
 +
 +    try {
 +      if (namespacePerms.add(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
 +              ZKSecurityTool.convertNamespacePermissions(namespacePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +
 +    // User had no system permission, nothing to revoke.
 +    if (sysPermBytes == null)
 +      return;
 +
 +    Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
 +
 +    try {
 +      if (sysPerms.remove(permission)) {
 +        synchronized (zooCache) {
 +          zooCache.clear();
 +          ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
 +              NodeExistsPolicy.OVERWRITE);
 +        }
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
 +
 +    // User had no table permission, nothing to revoke.
 +    if (serializedPerms == null)
 +      return;
 +
 +    Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
 +    try {
 +      if (tablePerms.remove(permission)) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        if (tablePerms.size() == 0)
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
 +        else
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
 +    byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
 +
 +    // User had no namespace permission, nothing to revoke.
 +    if (serializedPerms == null)
 +      return;
 +
 +    Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
 +    try {
 +      if (namespacePerms.remove(permission)) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        if (namespacePerms.size() == 0)
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
 +        else
 +          zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
 +              NodeExistsPolicy.OVERWRITE);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanTablePermissions(String table) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        for (String user : zooCache.getChildren(ZKUserPath))
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        zooCache.clear();
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        for (String user : zooCache.getChildren(ZKUserPath))
 +          zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
 +      }
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    // create the root user with all system privileges, no table privileges, and no record-level authorizations
 +    Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
 +    for (SystemPermission p : SystemPermission.values())
 +      rootPerms.add(p);
 +    Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
 +    // Allow the root user to flush the system tables
 +    tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
 +    // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
 +    Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
 +    namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
 +    namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
 +
 +    try {
 +      // prep parent node of users with root username
 +      if (!zoo.exists(ZKUserPath))
-         zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++        zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +      initUser(rootuser);
 +      zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
 +      for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
 +        createTablePerm(rootuser, entry.getKey(), entry.getValue());
 +      for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
 +        createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void initUser(String user) throws AccumuloSecurityException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
 +      zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
 +      zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
 +          ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  /**
 +   * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
 +   */
 +  private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
 +    synchronized (zooCache) {
 +      zooCache.clear();
 +      ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
 +          ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  @Override
 +  public void cleanUser(String user) throws AccumuloSecurityException {
 +    try {
 +      synchronized (zooCache) {
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
 +        zooCache.clear(ZKUserPath + "/" + user);
 +      }
 +    } catch (InterruptedException e) {
 +      log.error(e, e);
 +      throw new RuntimeException(e);
 +    } catch (KeeperException e) {
 +      log.error(e, e);
 +      if (e.code().equals(KeeperException.Code.NONODE))
 +        throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
 +
 +    }
 +  }
 +
 +  @Override
 +  public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] perms;
 +    try {
 +      String path = ZKUserPath + "/" + user + ZKUserSysPerms;
 +      ZooReaderWriter.getInstance().sync(path);
 +      perms = ZooReaderWriter.getInstance().getData(path, null);
 +    } catch (KeeperException e) {
 +      if (e.code() == Code.NONODE) {
 +        return false;
 +      }
 +      log.warn("Unhandled KeeperException, failing closed for table permission check", e);
 +      return false;
 +    } catch (InterruptedException e) {
 +      log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
 +      return false;
 +    }
 +
 +    if (perms == null)
 +      return false;
 +    return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
 +  }
 +
 +  @Override
 +  public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
 +    byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
 +    if (perms == null)
 +      return false;
 +    return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
 +  }
 +
 +  @Override
 +  public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
 +    return true;
 +  }
 +
 +  @Override
 +  public void initTable(String table) throws AccumuloSecurityException {
 +    // All proper housekeeping is done on delete and permission granting, no work needs to be done here
 +  }
 +}


[44/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
----------------------------------------------------------------------
diff --cc shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
index 934a41d,0000000..c3af70a
mode 100644,000000..100644
--- a/shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
@@@ -1,67 -1,0 +1,67 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
 +import static org.junit.Assert.*;
 +
 +import java.io.File;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.shell.ShellUtil;
 +import org.apache.commons.io.FileUtils;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Rule;
 +import org.junit.Test;
 +import org.junit.rules.TemporaryFolder;
 +
 +import com.google.common.collect.ImmutableList;
 +
 +public class ShellUtilTest {
 +
 +  @Rule
 +  public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
 +
 +  // String with 3 lines, with one empty line
 +  private static final String FILEDATA = "line1\n\nline2";
 +
 +  @Test
 +  public void testWithoutDecode() throws IOException {
 +    File testFile = new File(folder.getRoot(), "testFileNoDecode.txt");
 +    FileUtils.writeStringToFile(testFile, FILEDATA);
 +    List<Text> output = ShellUtil.scanFile(testFile.getAbsolutePath(), false);
 +    assertEquals(ImmutableList.of(new Text("line1"), new Text("line2")), output);
 +  }
 +
 +  @Test
 +  public void testWithDecode() throws IOException {
 +    File testFile = new File(folder.getRoot(), "testFileWithDecode.txt");
 +    FileUtils.writeStringToFile(testFile, FILEDATA);
 +    List<Text> output = ShellUtil.scanFile(testFile.getAbsolutePath(), true);
 +    assertEquals(
-         ImmutableList.of(new Text(Base64.decodeBase64("line1".getBytes(StandardCharsets.UTF_8))), new Text(Base64.decodeBase64("line2".getBytes(StandardCharsets.UTF_8)))),
++        ImmutableList.of(new Text(Base64.decodeBase64("line1".getBytes(UTF_8))), new Text(Base64.decodeBase64("line2".getBytes(UTF_8)))),
 +        output);
 +  }
 +
 +  @Test(expected = FileNotFoundException.class)
 +  public void testWithMissingFile() throws FileNotFoundException {
 +    ShellUtil.scanFile("missingFile.txt", false);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
----------------------------------------------------------------------
diff --cc shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
index e5b7394,0000000..0229cee
mode 100644,000000..100644
--- a/shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
@@@ -1,177 -1,0 +1,177 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.format;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.createNiceMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
- import java.nio.charset.StandardCharsets;
 +import java.util.Collections;
 +import java.util.Map;
 +import java.util.TreeMap;
 +
 +import jline.UnsupportedTerminal;
 +import jline.console.ConsoleReader;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.format.DeleterFormatter;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class DeleterFormatterTest {
 +  DeleterFormatter formatter;
 +  Map<Key,Value> data;
 +  BatchWriter writer;
 +  BatchWriter exceptionWriter;
 +  Shell shellState;
 +
 +  ByteArrayOutputStream baos;
 +  ConsoleReader reader;
 +
 +  SettableInputStream input;
 +
 +  class SettableInputStream extends InputStream {
 +    ByteArrayInputStream bais;
 +
 +    @Override
 +    public int read() throws IOException {
 +      return bais.read();
 +    }
 +
 +    public void set(String in) {
-       bais = new ByteArrayInputStream(in.getBytes(StandardCharsets.UTF_8));
++      bais = new ByteArrayInputStream(in.getBytes(UTF_8));
 +    }
 +  };
 +
 +  @Before
 +  public void setUp() throws IOException, MutationsRejectedException {
 +    input = new SettableInputStream();
 +    baos = new ByteArrayOutputStream();
 +
 +    MutationsRejectedException mre = createMock(MutationsRejectedException.class);
 +
 +    writer = createNiceMock(BatchWriter.class);
 +    exceptionWriter = createNiceMock(BatchWriter.class);
 +    exceptionWriter.close();
 +    expectLastCall().andThrow(mre);
 +    exceptionWriter.addMutation(anyObject(Mutation.class));
 +    expectLastCall().andThrow(mre);
 +
 +    shellState = createNiceMock(Shell.class);
 +
 +    reader = new ConsoleReader(input, baos, new UnsupportedTerminal());
 +    expect(shellState.getReader()).andReturn(reader).anyTimes();
 +
 +    replay(writer, exceptionWriter, shellState);
 +
 +    data = new TreeMap<Key,Value>();
-     data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(StandardCharsets.UTF_8)));
++    data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF_8)));
 +  }
 +
 +  @Test
 +  public void testEmpty() {
 +    formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true);
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testSingle() throws IOException {
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[DELETED]", " r ", "cf", "cq", "value");
 +  }
 +
 +  @Test
 +  public void testNo() throws IOException {
 +    input.set("no\n");
-     data.put(new Key("z"), new Value("v2".getBytes(StandardCharsets.UTF_8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[SKIPPED]", " r ", "cf", "cq", "value");
 +
 +    assertTrue(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testNoConfirmation() throws IOException {
 +    input.set("");
-     data.put(new Key("z"), new Value("v2".getBytes(StandardCharsets.UTF_8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[SKIPPED]", " r ", "cf", "cq", "value");
 +
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testYes() throws IOException {
 +    input.set("y\nyes\n");
-     data.put(new Key("z"), new Value("v2".getBytes(StandardCharsets.UTF_8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    verify("[DELETED]", " r ", "cf", "cq", "value");
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    verify("[DELETED]", " z ", "v2");
 +  }
 +
 +  @Test
 +  public void testMutationException() {
 +    formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  private void verify(String... chunks) throws IOException {
 +    reader.flush();
 +
 +    String output = baos.toString();
 +    for (String chunk : chunks) {
 +      assertTrue(output.contains(chunk));
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
index 93c2e98,b6f5214..8693d03
--- a/test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
+++ b/test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.List;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
index abf31d2,10ee8a4..c9d18e1
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Iterator;
  import java.util.Map.Entry;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
index d9527c5,c52ed29..1e1006b
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
index a73199c,51405db..c8e821b
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.Iterator;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
index a5ac0e8,3c20858..7b373c4
--- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Iterator;
  import java.util.Map.Entry;
  import java.util.Random;
@@@ -83,9 -84,9 +84,9 @@@ public class TestBinaryRows 
    public static void runTest(Connector connector, Opts opts, BatchWriterOpts bwOpts, ScannerOpts scanOpts) throws Exception {
      
      final Text CF = new Text("cf"), CQ = new Text("cq");
-     final byte[] CF_BYTES = "cf".getBytes(StandardCharsets.UTF_8), CQ_BYTES = "cq".getBytes(StandardCharsets.UTF_8);
+     final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
      if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
 -      BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
 +      BatchWriter bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
        boolean delete = opts.mode.equals("delete");
        
        for (long i = 0; i < opts.num; i++) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/TestIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestIngest.java
index 0548f4c,c3e77af..7f6c514
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Map.Entry;
  import java.util.Random;
  import java.util.Set;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index 133a5a5,16f0b3f..c5fcf34
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
index d6a16df,4404e2b..b986330
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.IOException;
  import java.io.InputStreamReader;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
index 9bf6343,cb32b6e..797413f
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Random;
  import java.util.Set;
  import java.util.UUID;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
index f7393d9,9f2ba34..dcc3e49
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.continuous;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Map.Entry;
  import java.util.Random;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
index 77f74da,0e71bf3..60154df
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.continuous;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Iterator;
  import java.util.Map.Entry;
  import java.util.Random;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 515d4a3,34ee9d0..049f9b8
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.lang.reflect.Method;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.HashSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
index 262f7b0,4f88c1b..680ca92
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.IOException;
  import java.io.InputStreamReader;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
index 4758aa0,17b8269..b3aae46
--- a/test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedOutputStream;
  import java.io.FileOutputStream;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
index 82d8660,82f028f..cab3126
--- a/test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
@@@ -16,12 -16,14 +16,13 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.FileNotFoundException;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
- import java.nio.charset.StandardCharsets;
 -
  import org.apache.log4j.Logger;
  
  public class PrintScanTimeHistogram {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
index 29800b6,8bf2d97..0824948
--- a/test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.InputStreamReader;
- import java.nio.charset.StandardCharsets;
  import java.text.SimpleDateFormat;
  import java.util.Date;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
index ca17fc5,11d19c5..dffd6c3
--- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.continuous;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.FileInputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
index 8d5cc16,348fb02..fdc704d
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
  import java.io.FileOutputStream;
  import java.io.ObjectOutputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
index 5954171,e304ab6..e1be8e6
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index 3b1aeaf,949fc52..879128f
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.HashMap;
  import java.util.Random;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index e404749,72efee3..96eb214
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.performance.metadata;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.HashSet;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
index 353a02b,e4ce3f3..93a8f61
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.randomwalk;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
index 6338d29,67438d9..e13e7d2
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
@@@ -16,20 -16,19 +16,20 @@@
   */
  package org.apache.accumulo.test.randomwalk.bulk;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.test.randomwalk.Environment;
  import org.apache.accumulo.test.randomwalk.State;
  
  public class BulkMinusOne extends BulkImportTest {
  
-   private static final Value negOne = new Value("-1".getBytes(StandardCharsets.UTF_8));
+   private static final Value negOne = new Value("-1".getBytes(UTF_8));
  
    @Override
 -  protected void runLater(State state) throws Exception {
 +  protected void runLater(State state, Environment env) throws Exception {
      log.info("Decrementing");
 -    BulkPlusOne.bulkLoadLots(log, state, negOne);
 +    BulkPlusOne.bulkLoadLots(log, state, env, negOne);
    }
  
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
index 3c2be07,b4fa7ba..f7a727a
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.bulk;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Arrays;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
index 9e50b61,3dcff6e..09bf883
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.concurrent;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.List;
  import java.util.Properties;
  import java.util.Random;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
index eb60971,ade3e38..0a9d3b9
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.randomwalk.concurrent;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.List;
  import java.util.Properties;
  import java.util.Random;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
index 28da4f1,61d7c61..03f2f39
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.concurrent;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Properties;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
index 85340f5,b3519b6..c16160e
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.concurrent;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
index a7669f8,7db2f04..6ca524a
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.image;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.security.MessageDigest;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
index a1731d1,d53658b..d239495
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.image;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.security.MessageDigest;
  import java.util.Properties;
  import java.util.Random;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
index 4a0515c,5dd3f8b..c3c91c0
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.multitable;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.security.MessageDigest;
  import java.util.ArrayList;
  import java.util.Properties;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
index f9ba5ba,07834b0..ba1893f
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.randomwalk.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Map;
  
  import org.apache.accumulo.core.client.Connector;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
index ef4593c,fa64e92..2499af2
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.security;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Iterator;
  import java.util.Map.Entry;
  import java.util.Properties;
@@@ -91,8 -91,8 +92,8 @@@ public class TableOp extends Test 
            }
            if (!canRead && !ambiguousZone)
              throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName);
 -          for (Entry<String,Integer> entry : WalkingSecurity.get(state).getAuthsMap().entrySet()) {
 +          for (Entry<String,Integer> entry : WalkingSecurity.get(state,env).getAuthsMap().entrySet()) {
-             if (auths.contains(entry.getKey().getBytes(StandardCharsets.UTF_8)))
+             if (auths.contains(entry.getKey().getBytes(UTF_8)))
                seen = seen - entry.getValue();
            }
            if (seen != 0 && !ambiguousAuths)
@@@ -143,13 -143,13 +144,13 @@@
          break;
        }
        case WRITE:
 -        boolean canWrite = WalkingSecurity.get(state).canWrite(WalkingSecurity.get(state).getTabCredentials(), tableName, namespaceName);
 -        boolean ambiguousZone = WalkingSecurity.get(state).inAmbiguousZone(conn.whoami(), tp);
 +        boolean canWrite = WalkingSecurity.get(state,env).canWrite(WalkingSecurity.get(state,env).getTabCredentials(), tableName, namespaceName);
 +        boolean ambiguousZone = WalkingSecurity.get(state,env).inAmbiguousZone(conn.whoami(), tp);
          
 -        String key = WalkingSecurity.get(state).getLastKey() + "1";
 +        String key = WalkingSecurity.get(state,env).getLastKey() + "1";
          Mutation m = new Mutation(new Text(key));
 -        for (String s : WalkingSecurity.get(state).getAuthsArray()) {
 +        for (String s : WalkingSecurity.get(state,env).getAuthsArray()) {
-           m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(StandardCharsets.UTF_8)));
+           m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(UTF_8)));
          }
          BatchWriter writer = null;
          try {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
index a0fdd9d,1fc0850..1795434
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.sequential;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Properties;
  
  import org.apache.accumulo.core.client.BatchWriter;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
index 0f04afc,b607e8d..5959105
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.randomwalk.shard;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedOutputStream;
  import java.io.IOException;
  import java.io.PrintStream;
@@@ -162,11 -162,11 +163,11 @@@ public class BulkInsert extends Test 
      }
    }
    
 -  private void sort(State state, FileSystem fs, String tableName, String seqFile, String outputDir, String workDir, int maxSplits) throws Exception {
 +  private void sort(State state, Environment env, FileSystem fs, String tableName, String seqFile, String outputDir, String workDir, int maxSplits) throws Exception {
      
-     PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDir + "/splits.txt"))), false, StandardCharsets.UTF_8.name());
+     PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDir + "/splits.txt"))), false, UTF_8.name());
      
 -    Connector conn = state.getConnector();
 +    Connector conn = env.getConnector();
      
      Collection<Text> splits = conn.tableOperations().listSplits(tableName, maxSplits);
      for (Text split : splits)

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
index f625870,d9bd7f9..f5ede55
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.test.randomwalk.shard;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.InputStreamReader;
- import java.nio.charset.StandardCharsets;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
index 76a20a0,b34defe..238a88d
--- a/test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
+++ b/test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.scalability;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Random;
  import java.util.UUID;
  import java.util.concurrent.TimeUnit;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
index d496ded,49b5d70..a173f79
--- a/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.test;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertTrue;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
index 85aec59,0000000..7ebcdd7
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
@@@ -1,283 -1,0 +1,284 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.util.HashMap;
 +import java.util.Map.Entry;
 +import java.util.Map;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.AccumuloServerException;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.junit.Assert;
 +import org.junit.Test;
 +
 +public class LargeSplitRowIT extends ConfigurableMacIT {
 +  static private final Logger log = Logger.getLogger(LargeSplitRowIT.class);
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setNumTservers(1);
 +
 +    Map<String,String> siteConfig = new HashMap<String,String>();
 +    siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
 +    cfg.setSiteConfig(siteConfig);
 +  }
 +
 +  // User added split
 +  @Test(timeout = 60 * 1000)
 +  public void userAddedSplit() throws Exception {
 +
 +    log.info("User added split");
 +
 +    // make a table and lower the TABLE_END_ROW_MAX_SIZE property
 +    final String tableName = getUniqueNames(1)[0];
 +    final Connector conn = getConnector();
 +    conn.tableOperations().create(tableName);
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
 +
 +    // Create a BatchWriter and add a mutation to the table
 +    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m = new Mutation("Row");
 +    m.put("cf", "cq", "value");
 +    batchWriter.addMutation(m);
 +    batchWriter.close();
 +
 +    // Create a split point that is too large to be an end row and fill it with all 'm'
 +    SortedSet<Text> partitionKeys = new TreeSet<Text>();
 +    byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
 +    for (int i = 0; i < data.length; i++) {
 +      data[i] = 'm';
 +    }
 +    partitionKeys.add(new Text(data));
 +
 +    // try to add the split point that is too large, if the split point is created the test fails.
 +    try {
 +      conn.tableOperations().addSplits(tableName, partitionKeys);
 +      Assert.fail();
 +    } catch (AccumuloServerException e) {}
 +
 +    // Make sure that the information that was written to the table before we tried to add the split point is still correct
 +    int counter = 0;
 +    final Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    for (Entry<Key,Value> entry : scanner) {
 +      counter++;
 +      Key k = entry.getKey();
 +      Assert.assertEquals("Row", k.getRow().toString());
 +      Assert.assertEquals("cf", k.getColumnFamily().toString());
 +      Assert.assertEquals("cq", k.getColumnQualifier().toString());
 +      Assert.assertEquals("value", entry.getValue().toString());
 +
 +    }
 +    // Make sure there is only one line in the table
 +    Assert.assertEquals(1, counter);
 +  }
 +
 +  // Test tablet server split with 250 entries with all the same prefix
 +  @Test(timeout = 60 * 1000)
 +  public void automaticSplitWith250Same() throws Exception {
 +    log.info("Automatic with 250 with same prefix");
 +
 +    // make a table and lower the configure properties
 +    final String tableName = getUniqueNames(1)[0];
 +    final Connector conn = getConnector();
 +    conn.tableOperations().create(tableName);
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
 +
 +    // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
 +    // Fill this key with all m's except the last spot
 +    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
 +    for (int i = 0; i < data.length - 1; i++) {
 +      data[i] = (byte) 'm';
 +    }
 +
 +    // Make the last place in the key different for every entry added to the table
 +    for (int i = 0; i < 250; i++) {
 +      data[data.length - 1] = (byte) i;
 +      Mutation m = new Mutation(data);
 +      m.put("cf", "cq", "value");
 +      batchWriter.addMutation(m);
 +    }
 +    // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
 +    batchWriter.close();
 +    conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
 +    Thread.sleep(500);
 +
 +    // Make sure all the data that was put in the table is still correct
 +    int count = 0;
 +    final Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    for (Entry<Key,Value> entry : scanner) {
 +      Key k = entry.getKey();
 +      data[data.length - 1] = (byte) count;
-       String expected = new String(data, StandardCharsets.UTF_8);
++      String expected = new String(data, UTF_8);
 +      Assert.assertEquals(expected, k.getRow().toString());
 +      Assert.assertEquals("cf", k.getColumnFamily().toString());
 +      Assert.assertEquals("cq", k.getColumnQualifier().toString());
 +      Assert.assertEquals("value", entry.getValue().toString());
 +      count++;
 +    }
 +    Assert.assertEquals(250, count);
 +
 +    // Make sure no splits occurred in the table
 +    Assert.assertEquals(0, conn.tableOperations().listSplits(tableName).size());
 +  }
 +
 +  // 10 0's; 10 2's; 10 4's... 10 30's etc
 +  @Test(timeout = 60 * 1000)
 +  public void automaticSplitWithGaps() throws Exception {
 +    log.info("Automatic Split With Gaps");
 +
 +    automaticSplit(30, 2);
 +  }
 +
 +  // 10 0's; 10 1's; 10 2's... 10 15's etc
 +  @Test(timeout = 60 * 1000)
 +  public void automaticSplitWithoutGaps() throws Exception {
 +    log.info("Automatic Split Without Gaps");
 +
 +    automaticSplit(15, 1);
 +  }
 +
 +  @Test(timeout = 60 * 1000)
 +  public void automaticSplitLater() throws Exception {
 +    log.info("Split later");
 +    automaticSplit(15, 1);
 +
 +    final Connector conn = getConnector();
 +
 +    String tableName = new String();
 +    java.util.Iterator<String> iterator = conn.tableOperations().list().iterator();
 +
 +    while (iterator.hasNext()) {
 +      String curr = iterator.next();
 +      if (!curr.equals("accumulo.metadata") && !curr.equals("accumulo.root")) {
 +        tableName = curr;
 +      }
 +    }
 +
 +    // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
 +    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    byte data[] = new byte[10];
 +
 +    // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value
 +    for (int j = 15; j < 150; j += 1) {
 +      for (int i = 0; i < data.length - 1; i++) {
 +        data[i] = (byte) j;
 +      }
 +
 +      for (int i = 0; i < 25; i++) {
 +        data[data.length - 1] = (byte) i;
 +        Mutation m = new Mutation(data);
 +        m.put("cf", "cq", "value");
 +        batchWriter.addMutation(m);
 +      }
 +    }
 +    // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
 +    batchWriter.close();
 +    conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
 +
 +    // Make sure a split occurs
 +    while (conn.tableOperations().listSplits(tableName).size() == 0) {
 +      Thread.sleep(250);
 +    }
 +
 +    Assert.assertTrue(0 < conn.tableOperations().listSplits(tableName).size());
 +  }
 +
 +  private void automaticSplit(int max, int spacing) throws Exception {
 +    // make a table and lower the configure properties
 +    final String tableName = getUniqueNames(1)[0];
 +    final Connector conn = getConnector();
 +    conn.tableOperations().create(tableName);
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
 +
 +    // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
 +    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
 +
 +    // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value
 +    for (int j = 0; j < max; j += spacing) {
 +      for (int i = 0; i < data.length - 1; i++) {
 +        data[i] = (byte) j;
 +      }
 +
 +      for (int i = 0; i < 10; i++) {
 +        data[data.length - 1] = (byte) i;
 +        Mutation m = new Mutation(data);
 +        m.put("cf", "cq", "value");
 +        batchWriter.addMutation(m);
 +      }
 +    }
 +    // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
 +    batchWriter.close();
 +    conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
 +    Thread.sleep(500);
 +
 +    // Make sure all the data that was put in the table is still correct
 +    int count = 0;
 +    int extra = 10;
 +    final Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (extra == 10) {
 +        extra = 0;
 +        for (int i = 0; i < data.length - 1; i++) {
 +          data[i] = (byte) count;
 +        }
 +        count += spacing;
 +
 +      }
 +      Key k = entry.getKey();
 +      data[data.length - 1] = (byte) extra;
-       String expected = new String(data, StandardCharsets.UTF_8);
++      String expected = new String(data, UTF_8);
 +      Assert.assertEquals(expected, k.getRow().toString());
 +      Assert.assertEquals("cf", k.getColumnFamily().toString());
 +      Assert.assertEquals("cq", k.getColumnQualifier().toString());
 +      Assert.assertEquals("value", entry.getValue().toString());
 +      extra++;
 +    }
 +    Assert.assertEquals(10, extra);
 +    Assert.assertEquals(max, count);
 +
 +    // Make sure no splits occured in the table
 +    Assert.assertEquals(0, conn.tableOperations().listSplits(tableName).size());
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index e9de27c,8188deb..5e885b5
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.test;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertFalse;
  import static org.junit.Assert.assertNotNull;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index d58c263,d5c940d..6025cf2
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.test;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertTrue;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
index 5901326,05de342..0d26445
--- a/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collection;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
index 1dfb3ce,9c4492e..311e354
--- a/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
@@@ -16,9 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  
- import java.nio.charset.StandardCharsets;
  import java.util.EnumSet;
  import java.util.Map.Entry;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index 3fabf49,688a326..9f00f74
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
index 3de5cda,465936e..6a9171a
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Iterator;
  import java.util.Map.Entry;
  import java.util.Random;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 4f5fc7d,ffe55bd..80ee990
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Iterator;
  import java.util.Map.Entry;
  import java.util.SortedSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
index 355a7bf,92bd714..3e5fec4
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.EnumSet;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
index b4067d3,7e5944e..b987dec
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.HashMap;
  import java.util.Iterator;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
index 67ecf10,5b5249b..0cdebeb
--- a/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
index 3877e0e,0578ef4..f7f8700
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
@@@ -16,9 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.*;
  
- import java.nio.charset.StandardCharsets;
  import java.util.Collections;
  import java.util.Map.Entry;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
index 02ab695,d35ba9f..ef64edc
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
@@@ -16,9 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertTrue;
  
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
index f0e4643,d77d060..91b19f6
--- a/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.Map.Entry;
  import java.util.Random;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 0512998,f4bee97..f4101de
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertFalse;
  import static org.junit.Assert.assertNotNull;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index 9ae94a3,189a55c..df18ef8
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
index 50267dc,90b881c..3514fb3
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Map.Entry;
  import java.util.TreeSet;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
index 760fdf7,3547b68..d9c3c77
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index 8deb51d,43d3a86..5223439
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@@ -16,9 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.HashSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
index 0dd511d,fccc79f..b75cccc
--- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
@@@ -16,9 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.test.functional;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  
- import java.nio.charset.StandardCharsets;
  import java.util.HashMap;
  import java.util.Map;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
index be1a45f,f2460cf..e0d66ce
--- a/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
index 0f4cf5b,0000000..554b08e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
@@@ -1,114 -1,0 +1,115 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.replication;
 +
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.replication.ReplicationConstants;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.hadoop.conf.Configuration;
 +import org.junit.Assert;
 +import org.junit.Test;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.collect.Iterables;
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * 
 + */
 +public class MultiTserverReplicationIT extends ConfigurableMacIT {
 +  private static final Logger log = LoggerFactory.getLogger(MultiTserverReplicationIT.class);
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setNumTservers(2);
 +  }
 +
 +  @Test
 +  public void tserverReplicationServicePortsAreAdvertised() throws Exception {
 +    // Wait for the cluster to be up
 +    Connector conn = getConnector();
 +    Instance inst = conn.getInstance();
 +
 +    // Wait for a tserver to come up to fulfill this request
 +    conn.tableOperations().create("foo");
 +    Scanner s = conn.createScanner("foo", Authorizations.EMPTY);
 +    Assert.assertEquals(0, Iterables.size(s));
 +
 +    ZooReader zreader = new ZooReader(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut());
 +    Set<String> tserverHost = new HashSet<>();
 +    tserverHost.addAll(zreader.getChildren(ZooUtil.getRoot(inst) + Constants.ZTSERVERS));
 +
 +    Set<HostAndPort> replicationServices = new HashSet<>();
 +
 +    for (String tserver : tserverHost) {
 +      try {
 +        byte[] portData = zreader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver, null);
-         HostAndPort replAddress = HostAndPort.fromString(new String(portData, StandardCharsets.UTF_8));
++        HostAndPort replAddress = HostAndPort.fromString(new String(portData, UTF_8));
 +        replicationServices.add(replAddress);
 +      } catch (Exception e) {
 +        log.error("Could not find port for {}", tserver, e);
 +        Assert.fail("Did not find replication port advertisement for " + tserver);
 +      }
 +    }
 +
 +    // Each tserver should also have equial replicaiton services running internally
 +    Assert.assertEquals("Expected an equal number of replication servicers and tservers", tserverHost.size(), replicationServices.size());
 +  }
 +
 +  @Test
 +  public void masterReplicationServicePortsAreAdvertised() throws Exception {
 +    // Wait for the cluster to be up
 +    Connector conn = getConnector();
 +    Instance inst = conn.getInstance();
 +
 +    // Wait for a tserver to come up to fulfill this request
 +    conn.tableOperations().create("foo");
 +    Scanner s = conn.createScanner("foo", Authorizations.EMPTY);
 +    Assert.assertEquals(0, Iterables.size(s));
 +
 +    ZooReader zreader = new ZooReader(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut());
 +
 +    // Should have one master instance
 +    Assert.assertEquals(1, inst.getMasterLocations().size());
 +
 +    // Get the master thrift service addr
 +    String masterAddr = Iterables.getOnlyElement(inst.getMasterLocations());
 +
 +    // Get the master replication coordinator addr
-     String replCoordAddr = new String(zreader.getData(ZooUtil.getRoot(inst) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR, null), StandardCharsets.UTF_8);
++    String replCoordAddr = new String(zreader.getData(ZooUtil.getRoot(inst) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR, null), UTF_8);
 +
 +    // They shouldn't be the same
 +    Assert.assertNotEquals(masterAddr, replCoordAddr);
 +
 +    // Neither should be zero as the port
 +    Assert.assertNotEquals(0, HostAndPort.fromString(masterAddr).getPort());
 +    Assert.assertNotEquals(0, HostAndPort.fromString(replCoordAddr).getPort());
 +  }
 +}


[36/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index e8d5bbf,0000000..002389a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@@ -1,399 -1,0 +1,400 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP;
 +
 +import java.nio.ByteBuffer;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +import org.apache.zookeeper.KeeperException.NotEmptyException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class LiveTServerSet implements Watcher {
 +
 +  public interface Listener {
 +    void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added);
 +  }
 +
 +  private static final Logger log = Logger.getLogger(LiveTServerSet.class);
 +
 +  private final Listener cback;
 +  private final Instance instance;
 +  private final AccumuloConfiguration conf;
 +  private ZooCache zooCache;
 +
 +  public class TServerConnection {
 +    private final HostAndPort address;
 +
 +    public TServerConnection(HostAndPort addr) throws TException {
 +      address = addr;
 +    }
 +
 +    private String lockString(ZooLock mlock) {
 +      return mlock.getLockID().serialize(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK);
 +    }
 +
 +    public void assignTablet(ZooLock lock, KeyExtent extent) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.loadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void unloadTablet(ZooLock lock, KeyExtent extent, boolean save) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.unloadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift(), save);
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public TabletServerStatus getTableMap(boolean usePooledConnection) throws TException, ThriftSecurityException {
 +
 +      if (usePooledConnection == true)
 +        throw new UnsupportedOperationException();
 +
 +      TTransport transport = ThriftUtil.createTransport(address, conf);
 +
 +      try {
 +        TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
 +        return client.getTabletServerStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
 +      } finally {
 +        if (transport != null)
 +          transport.close();
 +      }
 +    }
 +
 +    public void halt(ZooLock lock) throws TException, ThriftSecurityException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.halt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void fastHalt(ZooLock lock) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.fastHalt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void flush(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.flush(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
 +            startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void chop(ZooLock lock, KeyExtent extent) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.chop(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void splitTablet(ZooLock lock, KeyExtent extent, Text splitPoint) throws TException, ThriftSecurityException, NotServingTabletException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.splitTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), extent.toThrift(),
 +            ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength()));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void flushTablet(ZooLock lock, KeyExtent extent) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.flushTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public void compact(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        client.compact(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
 +            startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +    public boolean isActive(long tid) throws TException {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
 +      try {
 +        return client.isActive(Tracer.traceInfo(), tid);
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    }
 +
 +  }
 +
 +  static class TServerInfo {
 +    TServerConnection connection;
 +    TServerInstance instance;
 +
 +    TServerInfo(TServerInstance instance, TServerConnection connection) {
 +      this.connection = connection;
 +      this.instance = instance;
 +    }
 +  };
 +
 +  // The set of active tservers with locks, indexed by their name in zookeeper
 +  private Map<String,TServerInfo> current = new HashMap<String,TServerInfo>();
 +  // as above, indexed by TServerInstance
 +  private Map<TServerInstance,TServerInfo> currentInstances = new HashMap<TServerInstance,TServerInfo>();
 +
 +  // The set of entries in zookeeper without locks, and the first time each was noticed
 +  private Map<String,Long> locklessServers = new HashMap<String,Long>();
 +
 +  public LiveTServerSet(Instance instance, AccumuloConfiguration conf, Listener cback) {
 +    this.cback = cback;
 +    this.instance = instance;
 +    this.conf = conf;
 +
 +  }
 +
 +  public synchronized ZooCache getZooCache() {
 +    if (zooCache == null)
 +      zooCache = new ZooCache(this);
 +    return zooCache;
 +  }
 +
 +  public synchronized void startListeningForTabletServerChanges() {
 +    scanServers();
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        scanServers();
 +      }
 +    }, 0, 5000);
 +  }
 +
 +  public synchronized void scanServers() {
 +    try {
 +      final Set<TServerInstance> updates = new HashSet<TServerInstance>();
 +      final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
 +
 +      final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +
 +      HashSet<String> all = new HashSet<String>(current.keySet());
 +      all.addAll(getZooCache().getChildren(path));
 +
 +      locklessServers.keySet().retainAll(all);
 +
 +      for (String zPath : all) {
 +        checkServer(updates, doomed, path, zPath);
 +      }
 +
 +      // log.debug("Current: " + current.keySet());
 +      if (!doomed.isEmpty() || !updates.isEmpty())
 +        this.cback.update(this, doomed, updates);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +
 +  private void deleteServerNode(String serverNode) throws InterruptedException, KeeperException {
 +    try {
 +      ZooReaderWriter.getInstance().delete(serverNode, -1);
 +    } catch (NotEmptyException ex) {
 +      // race condition: tserver created the lock after our last check; we'll see it at the next check
 +    } catch (NoNodeException nne) {
 +      // someone else deleted it
 +    }
 +  }
 +
 +  private synchronized void checkServer(final Set<TServerInstance> updates, final Set<TServerInstance> doomed, final String path, final String zPath)
 +      throws TException, InterruptedException, KeeperException {
 +
 +    TServerInfo info = current.get(zPath);
 +
 +    final String lockPath = path + "/" + zPath;
 +    Stat stat = new Stat();
 +    byte[] lockData = ZooLock.getLockData(getZooCache(), lockPath, stat);
 +
 +    if (lockData == null) {
 +      if (info != null) {
 +        doomed.add(info.instance);
 +        current.remove(zPath);
 +        currentInstances.remove(info.instance);
 +      }
 +
 +      Long firstSeen = locklessServers.get(zPath);
 +      if (firstSeen == null) {
 +        locklessServers.put(zPath, System.currentTimeMillis());
 +      } else if (System.currentTimeMillis() - firstSeen > 10 * 60 * 1000) {
 +        deleteServerNode(path + "/" + zPath);
 +        locklessServers.remove(zPath);
 +      }
 +    } else {
 +      locklessServers.remove(zPath);
-       ServerServices services = new ServerServices(new String(lockData, Constants.UTF8));
++      ServerServices services = new ServerServices(new String(lockData, UTF_8));
 +      HostAndPort client = services.getAddress(ServerServices.Service.TSERV_CLIENT);
 +      TServerInstance instance = new TServerInstance(client, stat.getEphemeralOwner());
 +
 +      if (info == null) {
 +        updates.add(instance);
 +        TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
 +        current.put(zPath, tServerInfo);
 +        currentInstances.put(instance, tServerInfo);
 +      } else if (!info.instance.equals(instance)) {
 +        doomed.add(info.instance);
 +        updates.add(instance);
 +        TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
 +        current.put(zPath, tServerInfo);
 +        currentInstances.put(info.instance, tServerInfo);
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void process(WatchedEvent event) {
 +
 +    // its important that these event are propagated by ZooCache, because this ensures when reading zoocache that is has already processed the event and cleared
 +    // relevant nodes before code below reads from zoocache
 +
 +    if (event.getPath() != null) {
 +      if (event.getPath().endsWith(Constants.ZTSERVERS)) {
 +        scanServers();
 +      } else if (event.getPath().contains(Constants.ZTSERVERS)) {
 +        int pos = event.getPath().lastIndexOf('/');
 +
 +        // do only if ZTSERVER is parent
 +        if (pos >= 0 && event.getPath().substring(0, pos).endsWith(Constants.ZTSERVERS)) {
 +
 +          String server = event.getPath().substring(pos + 1);
 +
 +          final Set<TServerInstance> updates = new HashSet<TServerInstance>();
 +          final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
 +
 +          final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +
 +          try {
 +            checkServer(updates, doomed, path, server);
 +            if (!doomed.isEmpty() || !updates.isEmpty())
 +              this.cback.update(this, doomed, updates);
 +          } catch (Exception ex) {
 +            log.error(ex, ex);
 +          }
 +        }
 +      }
 +    }
 +  }
 +
 +  public synchronized TServerConnection getConnection(TServerInstance server) {
 +    if (server == null)
 +      return null;
 +    TServerInfo tServerInfo = currentInstances.get(server);
 +    if (tServerInfo == null)
 +      return null;
 +    return tServerInfo.connection;
 +  }
 +
 +  public synchronized Set<TServerInstance> getCurrentServers() {
 +    return new HashSet<TServerInstance>(currentInstances.keySet());
 +  }
 +
 +  public synchronized int size() {
 +    return current.size();
 +  }
 +
 +  public synchronized TServerInstance find(String tabletServer) {
 +    HostAndPort addr = AddressUtil.parseAddress(tabletServer, false);
 +    for (Entry<String,TServerInfo> entry : current.entrySet()) {
 +      if (entry.getValue().instance.getLocation().equals(addr))
 +        return entry.getValue().instance;
 +    }
 +    return null;
 +  }
 +
 +  public synchronized void remove(TServerInstance server) {
 +    String zPath = null;
 +    for (Entry<String,TServerInfo> entry : current.entrySet()) {
 +      if (entry.getValue().instance.equals(server)) {
 +        zPath = entry.getKey();
 +        break;
 +      }
 +    }
 +    if (zPath == null)
 +      return;
 +    current.remove(zPath);
 +    currentInstances.remove(server);
 +
 +    log.info("Removing zookeeper lock for " + server);
 +    String fullpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + zPath;
 +    try {
 +      ZooReaderWriter.getInstance().recursiveDelete(fullpath, SKIP);
 +    } catch (Exception e) {
 +      String msg = "error removing tablet server lock";
 +      log.fatal(msg, e);
 +      Halt.halt(msg, -1);
 +    }
 +    getZooCache().clear(fullpath);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
index 2f657c4,0000000..568e6f6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
@@@ -1,90 -1,0 +1,91 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.DeadServer;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.data.Stat;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +public class DeadServerList {
 +  private static final Logger log = Logger.getLogger(DeadServerList.class);
 +  private final String path;
 +  
 +  public DeadServerList(String path) {
 +    this.path = path;
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.mkdirs(path);
 +    } catch (Exception ex) {
 +      log.error("Unable to make parent directories of " + path, ex);
 +    }
 +  }
 +  
 +  public List<DeadServer> getList() {
 +    List<DeadServer> result = new ArrayList<DeadServer>();
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      List<String> children = zoo.getChildren(path);
 +      if (children != null) {
 +        for (String child : children) {
 +          Stat stat = new Stat();
 +          byte[] data;
 +          try {
 +            data = zoo.getData(path + "/" + child, stat);
 +          } catch (NoNodeException nne) {
 +            // Another thread or process can delete child while this loop is running.
 +            // We ignore this error since it's harmless if we miss the deleted server
 +            // in the dead server list.
 +            continue;
 +          }
-           DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, Constants.UTF8));
++          DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, UTF_8));
 +          result.add(server);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +    return result;
 +  }
 +  
 +  public void delete(String server) {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
 +      zoo.recursiveDelete(path + "/" + server, NodeMissingPolicy.SKIP);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +  
 +  public void post(String server, String cause) {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    try {
-       zoo.putPersistentData(path + "/" + server, cause.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++      zoo.putPersistentData(path + "/" + server, cause.getBytes(UTF_8), NodeExistsPolicy.SKIP);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
index 400d0ac,0000000..b4ef519
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
@@@ -1,143 -1,0 +1,144 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.ObjectInputStream;
 +import java.io.ObjectOutputStream;
 +import java.io.Serializable;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.hadoop.io.Text;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * A tablet is assigned to a tablet server at the given address as long as it is alive and well. When the tablet server is restarted, the instance information
 + * it advertises will change. Therefore tablet assignments can be considered out-of-date if the tablet server instance information has been changed.
 + * 
 + */
 +public class TServerInstance implements Comparable<TServerInstance>, Serializable {
 +  
 +  private static final long serialVersionUID = 1L;
 +  
 +  // HostAndPort is not Serializable
 +  private transient HostAndPort location;
 +  private String session;
 +  private String cachedStringRepresentation;
 +  
 +  public TServerInstance(HostAndPort address, String session) {
 +    this.location = address;
 +    this.session = session;
 +    this.cachedStringRepresentation = hostPort() + "[" + session + "]";
 +  }
 +  
 +  public TServerInstance(HostAndPort address, long session) {
 +    this(address, Long.toHexString(session));
 +  }
 +  
 +  public TServerInstance(String address, long session) {
 +    this(AddressUtil.parseAddress(address, false), Long.toHexString(session));
 +  }
 +  
 +  public TServerInstance(Value address, Text session) {
-     this(AddressUtil.parseAddress(new String(address.get(), Constants.UTF8), false), session.toString());
++    this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString());
 +  }
 +  
 +  public void putLocation(Mutation m) {
 +    m.put(TabletsSection.CurrentLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
 +  }
 +  
 +  public void putFutureLocation(Mutation m) {
 +    m.put(TabletsSection.FutureLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
 +  }
 +  
 +  public void putLastLocation(Mutation m) {
 +    m.put(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
 +  }
 +  
 +  public void clearLastLocation(Mutation m) {
 +    m.putDelete(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier());
 +  }
 +  
 +  @Override
 +  public int compareTo(TServerInstance other) {
 +    if (this == other)
 +      return 0;
 +    return this.toString().compareTo(other.toString());
 +  }
 +  
 +  @Override
 +  public int hashCode() {
 +    return toString().hashCode();
 +  }
 +  
 +  @Override
 +  public boolean equals(Object obj) {
 +    if (obj instanceof TServerInstance) {
 +      return compareTo((TServerInstance) obj) == 0;
 +    }
 +    return false;
 +  }
 +  
 +  @Override
 +  public String toString() {
 +    return cachedStringRepresentation;
 +  }
 +  
 +  public int port() {
 +    return getLocation().getPort();
 +  }
 +  
 +  public String host() {
 +    return getLocation().getHostText();
 +  }
 +  
 +  public String hostPort() {
 +    return getLocation().toString();
 +  }
 +  
 +  public Text asColumnQualifier() {
 +    return new Text(this.getSession());
 +  }
 +  
 +  public Value asMutationValue() {
-     return new Value(getLocation().toString().getBytes(Constants.UTF8));
++    return new Value(getLocation().toString().getBytes(UTF_8));
 +  }
 +  
 +  public HostAndPort getLocation() {
 +    return location;
 +  }
 +  
 +  public String getSession() {
 +    return session;
 +  }
 +  
 +  private void writeObject(ObjectOutputStream out) throws IOException {
 +    out.defaultWriteObject();
 +    out.writeObject(location.toString());
 +  }
 +  
 +  private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
 +    in.defaultReadObject();
 +    location = HostAndPort.fromString(in.readObject().toString());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
index f4d5591,0000000..91be9c3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
@@@ -1,189 -1,0 +1,190 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorEnvironment;
 +import org.apache.accumulo.core.iterators.SkippingIterator;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
 +import org.apache.hadoop.io.DataInputBuffer;
 +import org.apache.hadoop.io.DataOutputBuffer;
 +import org.apache.hadoop.io.Text;
 +
 +public class TabletStateChangeIterator extends SkippingIterator {
 +  
 +  private static final String SERVERS_OPTION = "servers";
 +  private static final String TABLES_OPTION = "tables";
 +  private static final String MERGES_OPTION = "merges";
 +  // private static final Logger log = Logger.getLogger(TabletStateChangeIterator.class);
 +  
 +  Set<TServerInstance> current;
 +  Set<String> onlineTables;
 +  Map<Text,MergeInfo> merges;
 +  
 +  @Override
 +  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
 +    super.init(source, options, env);
 +    current = parseServers(options.get(SERVERS_OPTION));
 +    onlineTables = parseTables(options.get(TABLES_OPTION));
 +    merges = parseMerges(options.get(MERGES_OPTION));
 +  }
 +  
 +  private Set<String> parseTables(String tables) {
 +    if (tables == null)
 +      return null;
 +    Set<String> result = new HashSet<String>();
 +    for (String table : tables.split(","))
 +      result.add(table);
 +    return result;
 +  }
 +  
 +  private Set<TServerInstance> parseServers(String servers) {
 +    if (servers == null)
 +      return null;
 +    // parse "host:port[INSTANCE]"
 +    Set<TServerInstance> result = new HashSet<TServerInstance>();
 +    if (servers.length() > 0) {
 +      for (String part : servers.split(",")) {
 +        String parts[] = part.split("\\[", 2);
 +        String hostport = parts[0];
 +        String instance = parts[1];
 +        if (instance != null && instance.endsWith("]"))
 +          instance = instance.substring(0, instance.length() - 1);
 +        result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance));
 +      }
 +    }
 +    return result;
 +  }
 +  
 +  private Map<Text,MergeInfo> parseMerges(String merges) {
 +    if (merges == null)
 +      return null;
 +    try {
 +      Map<Text,MergeInfo> result = new HashMap<Text,MergeInfo>();
 +      DataInputBuffer buffer = new DataInputBuffer();
-       byte[] data = Base64.decodeBase64(merges.getBytes(Constants.UTF8));
++      byte[] data = Base64.decodeBase64(merges.getBytes(UTF_8));
 +      buffer.reset(data, data.length);
 +      while (buffer.available() > 0) {
 +        MergeInfo mergeInfo = new MergeInfo();
 +        mergeInfo.readFields(buffer);
 +        result.put(mergeInfo.extent.getTableId(), mergeInfo);
 +      }
 +      return result;
 +    } catch (Exception ex) {
 +      throw new RuntimeException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  protected void consume() throws IOException {
 +    while (getSource().hasTop()) {
 +      Key k = getSource().getTopKey();
 +      Value v = getSource().getTopValue();
 +      
 +      if (onlineTables == null || current == null)
 +        return;
 +      
 +      TabletLocationState tls;
 +      try {
 +        tls = MetaDataTableScanner.createTabletLocationState(k, v);
 +        if (tls == null)
 +          return;
 +      } catch (BadLocationStateException e) {
 +        // maybe the master can do something with a tablet with bad/inconsistent state
 +        return;
 +      }
 +      // we always want data about merges
 +      MergeInfo merge = merges.get(tls.extent.getTableId());
 +      if (merge != null && merge.getExtent() != null && merge.getExtent().overlaps(tls.extent)) {
 +        return;
 +      }
 +      // is the table supposed to be online or offline?
 +      boolean shouldBeOnline = onlineTables.contains(tls.extent.getTableId().toString());
 +      
 +      switch (tls.getState(current)) {
 +        case ASSIGNED:
 +          // we always want data about assigned tablets
 +          return;
 +        case HOSTED:
 +          if (!shouldBeOnline)
 +            return;
 +        case ASSIGNED_TO_DEAD_SERVER:
 +          return;
 +        case UNASSIGNED:
 +          if (shouldBeOnline)
 +            return;
 +      }
 +      // table is in the expected state so don't bother returning any information about it
 +      getSource().next();
 +    }
 +  }
 +  
 +  @Override
 +  public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
 +    throw new UnsupportedOperationException();
 +  }
 +  
 +  public static void setCurrentServers(IteratorSetting cfg, Set<TServerInstance> goodServers) {
 +    if (goodServers != null) {
 +      List<String> servers = new ArrayList<String>();
 +      for (TServerInstance server : goodServers)
 +        servers.add(server.toString());
 +      cfg.addOption(SERVERS_OPTION, StringUtil.join(servers, ","));
 +    }
 +  }
 +  
 +  public static void setOnlineTables(IteratorSetting cfg, Set<String> onlineTables) {
 +    if (onlineTables != null)
 +      cfg.addOption(TABLES_OPTION, StringUtil.join(onlineTables, ","));
 +  }
 +  
 +  public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) {
 +    DataOutputBuffer buffer = new DataOutputBuffer();
 +    try {
 +      for (MergeInfo info : merges) {
 +        KeyExtent extent = info.getExtent();
 +        if (extent != null && !info.getState().equals(MergeState.NONE)) {
 +          info.write(buffer);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      throw new RuntimeException(ex);
 +    }
 +    String encoded = Base64.encodeBase64String(Arrays.copyOf(buffer.getData(), buffer.getLength()));
 +    cfg.addOption(MERGES_OPTION, encoded);
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
index b0ed03f,0000000..8f0c9c7
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
@@@ -1,97 -1,0 +1,98 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +
 +public class ZooStore implements DistributedStore {
 +  
 +  private static final Logger log = Logger.getLogger(ZooStore.class);
 +  
 +  String basePath;
 +  
 +  ZooCache cache = new ZooCache();
 +  
 +  public ZooStore(String basePath) throws IOException {
 +    if (basePath.endsWith("/"))
 +      basePath = basePath.substring(0, basePath.length() - 1);
 +    this.basePath = basePath;
 +  }
 +  
 +  public ZooStore() throws IOException {
 +    this(ZooUtil.getRoot(HdfsZooInstance.getInstance().getInstanceID()));
 +  }
 +  
 +  @Override
 +  public byte[] get(String path) throws DistributedStoreException {
 +    try {
 +      return cache.get(relative(path));
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  private String relative(String path) {
 +    return basePath + path;
 +  }
 +  
 +  @Override
 +  public List<String> getChildren(String path) throws DistributedStoreException {
 +    try {
 +      return cache.getChildren(relative(path));
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  public void put(String path, byte[] bs) throws DistributedStoreException {
 +    try {
 +      path = relative(path);
 +      ZooReaderWriter.getInstance().putPersistentData(path, bs, NodeExistsPolicy.OVERWRITE);
 +      cache.clear();
-       log.debug("Wrote " + new String(bs, Constants.UTF8) + " to " + path);
++      log.debug("Wrote " + new String(bs, UTF_8) + " to " + path);
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  public void remove(String path) throws DistributedStoreException {
 +    try {
 +      log.debug("Removing " + path);
 +      path = relative(path);
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      if (zoo.exists(path))
 +        zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
 +      cache.clear();
 +    } catch (Exception ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
index 0a0e1ee,0000000..15ade65
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
@@@ -1,179 -1,0 +1,180 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Iterator;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.commons.lang.NotImplementedException;
 +import org.apache.log4j.Logger;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class ZooTabletStateStore extends TabletStateStore {
 +  
 +  private static final Logger log = Logger.getLogger(ZooTabletStateStore.class);
 +  final private DistributedStore store;
 +  
 +  public ZooTabletStateStore(DistributedStore store) {
 +    this.store = store;
 +  }
 +  
 +  public ZooTabletStateStore() throws DistributedStoreException {
 +    try {
 +      store = new ZooStore();
 +    } catch (IOException ex) {
 +      throw new DistributedStoreException(ex);
 +    }
 +  }
 +  
 +  @Override
 +  public ClosableIterator<TabletLocationState> iterator() {
 +    return new ClosableIterator<TabletLocationState>() {
 +      boolean finished = false;
 +      
 +      @Override
 +      public boolean hasNext() {
 +        return !finished;
 +      }
 +      
 +      @Override
 +      public TabletLocationState next() {
 +        finished = true;
 +        try {
 +          byte[] future = store.get(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
 +          byte[] current = store.get(RootTable.ZROOT_TABLET_LOCATION);
 +          byte[] last = store.get(RootTable.ZROOT_TABLET_LAST_LOCATION);
 +          
 +          TServerInstance currentSession = null;
 +          TServerInstance futureSession = null;
 +          TServerInstance lastSession = null;
 +          
 +          if (future != null)
 +            futureSession = parse(future);
 +          
 +          if (last != null)
 +            lastSession = parse(last);
 +          
 +          if (current != null) {
 +            currentSession = parse(current);
 +            futureSession = null;
 +          }
 +          List<Collection<String>> logs = new ArrayList<Collection<String>>();
 +          for (String entry : store.getChildren(RootTable.ZROOT_TABLET_WALOGS)) {
 +            byte[] logInfo = store.get(RootTable.ZROOT_TABLET_WALOGS + "/" + entry);
 +            if (logInfo != null) {
 +              LogEntry logEntry = new LogEntry();
 +              logEntry.fromBytes(logInfo);
 +              logs.add(logEntry.logSet);
 +              log.debug("root tablet logSet " + logEntry.logSet);
 +            }
 +          }
 +          TabletLocationState result = new TabletLocationState(RootTable.EXTENT, futureSession, currentSession, lastSession, logs, false);
 +          log.debug("Returning root tablet state: " + result);
 +          return result;
 +        } catch (Exception ex) {
 +          throw new RuntimeException(ex);
 +        }
 +      }
 +      
 +      @Override
 +      public void remove() {
 +        throw new NotImplementedException();
 +      }
 +
 +      @Override
 +      public void close() {
 +      }
 +    };
 +  }
 +  
 +  protected TServerInstance parse(byte[] current) {
-     String str = new String(current, Constants.UTF8);
++    String str = new String(current, UTF_8);
 +    String[] parts = str.split("[|]", 2);
 +    HostAndPort address = HostAndPort.fromString(parts[0]);
 +    if (parts.length > 1 && parts[1] != null && parts[1].length() > 0) {
 +      return new TServerInstance(address, parts[1]);
 +    } else {
 +      // a 1.2 location specification: DO NOT WANT
 +      return null;
 +    }
 +  }
 +  
 +  @Override
 +  public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException {
 +    if (assignments.size() != 1)
 +      throw new IllegalArgumentException("There is only one root tablet");
 +    Assignment assignment = assignments.iterator().next();
 +    if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
 +      throw new IllegalArgumentException("You can only store the root tablet location");
 +    String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
 +    Iterator<TabletLocationState> currentIter = iterator();
 +    TabletLocationState current = currentIter.next();
 +    if (current.current != null) {
 +      throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
 +    }
-     store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(Constants.UTF8));
++    store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(UTF_8));
 +  }
 +  
 +  @Override
 +  public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException {
 +    if (assignments.size() != 1)
 +      throw new IllegalArgumentException("There is only one root tablet");
 +    Assignment assignment = assignments.iterator().next();
 +    if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
 +      throw new IllegalArgumentException("You can only store the root tablet location");
 +    String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
 +    Iterator<TabletLocationState> currentIter = iterator();
 +    TabletLocationState current = currentIter.next();
 +    if (current.current != null) {
 +      throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
 +    }
 +    if (!current.future.equals(assignment.server)) {
 +      throw new DistributedStoreException("Root tablet is already assigned to " + current.future);
 +    }
-     store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(Constants.UTF8));
-     store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(Constants.UTF8));
++    store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(UTF_8));
++    store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(UTF_8));
 +    // Make the following unnecessary by making the entire update atomic
 +    store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
 +    log.debug("Put down root tablet location");
 +  }
 +  
 +  @Override
 +  public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException {
 +    if (tablets.size() != 1)
 +      throw new IllegalArgumentException("There is only one root tablet");
 +    TabletLocationState tls = tablets.iterator().next();
 +    if (tls.extent.compareTo(RootTable.EXTENT) != 0)
 +      throw new IllegalArgumentException("You can only store the root tablet location");
 +    store.remove(RootTable.ZROOT_TABLET_LOCATION);
 +    store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
 +    log.debug("unassign root tablet location");
 +  }
 +  
 +  @Override
 +  public String name() {
 +    return "Root Table";
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 16b4bb8,0000000..409fa62
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,273 -1,0 +1,274 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.metrics;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.io.IOException;
 +import java.io.OutputStreamWriter;
 +import java.io.Writer;
 +import java.lang.management.ManagementFactory;
 +import java.text.SimpleDateFormat;
 +import java.util.Date;
 +import java.util.concurrent.ConcurrentHashMap;
 +
 +import javax.management.MBeanServer;
 +import javax.management.ObjectName;
 +import javax.management.StandardMBean;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.commons.lang.builder.ToStringBuilder;
 +import org.apache.commons.lang.builder.ToStringStyle;
 +import org.apache.commons.lang.time.DateUtils;
 +
 +public abstract class AbstractMetricsImpl {
 +  
 +  public class Metric {
 +    
 +    private long count = 0;
 +    private long avg = 0;
 +    private long min = 0;
 +    private long max = 0;
 +    
 +    public long getCount() {
 +      return count;
 +    }
 +    
 +    public long getAvg() {
 +      return avg;
 +    }
 +    
 +    public long getMin() {
 +      return min;
 +    }
 +    
 +    public long getMax() {
 +      return max;
 +    }
 +    
 +    public void incCount() {
 +      count++;
 +    }
 +    
 +    public void addAvg(long a) {
 +      if (a < 0)
 +        return;
 +      avg = (long) ((avg * .8) + (a * .2));
 +    }
 +    
 +    public void addMin(long a) {
 +      if (a < 0)
 +        return;
 +      min = Math.min(min, a);
 +    }
 +    
 +    public void addMax(long a) {
 +      if (a < 0)
 +        return;
 +      max = Math.max(max, a);
 +    }
 +    
 +    @Override
 +    public String toString() {
 +      return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
 +    }
 +    
 +  }
 +  
 +  static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
 +  
 +  private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
 +  
 +  private boolean currentlyLogging = false;
 +  
 +  private File logDir = null;
 +  
 +  private String metricsPrefix = null;
 +  
 +  private Date today = new Date();
 +  
 +  private File logFile = null;
 +  
 +  private Writer logWriter = null;
 +  
 +  private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
 +  
 +  private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
 +  
 +  private MetricsConfiguration config = null;
 +  
 +  public AbstractMetricsImpl() {
 +    this.metricsPrefix = getMetricsPrefix();
 +    config = new MetricsConfiguration(metricsPrefix);
 +  }
 +  
 +  /**
 +   * Registers a StandardMBean with the MBean Server
 +   */
 +  public void register(StandardMBean mbean) throws Exception {
 +    // Register this object with the MBeanServer
 +    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 +    if (null == getObjectName())
 +      throw new IllegalArgumentException("MBean object name must be set.");
 +    mbs.registerMBean(mbean, getObjectName());
 +    
 +    setupLogging();
 +  }
 +  
 +  /**
 +   * Registers this MBean with the MBean Server
 +   */
 +  public void register() throws Exception {
 +    // Register this object with the MBeanServer
 +    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 +    if (null == getObjectName())
 +      throw new IllegalArgumentException("MBean object name must be set.");
 +    mbs.registerMBean(this, getObjectName());
 +    setupLogging();
 +  }
 +  
 +  public void createMetric(String name) {
 +    registry.put(name, new Metric());
 +  }
 +  
 +  public Metric getMetric(String name) {
 +    return registry.get(name);
 +  }
 +  
 +  public long getMetricCount(String name) {
 +    return registry.get(name).getCount();
 +  }
 +  
 +  public long getMetricAvg(String name) {
 +    return registry.get(name).getAvg();
 +  }
 +  
 +  public long getMetricMin(String name) {
 +    return registry.get(name).getMin();
 +  }
 +  
 +  public long getMetricMax(String name) {
 +    return registry.get(name).getMax();
 +  }
 +  
 +  private void setupLogging() throws IOException {
 +    if (null == config.getMetricsConfiguration())
 +      return;
 +    // If we are already logging, then return
 +    if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +      // Check to see if directory exists, else make it
 +      String mDir = config.getMetricsConfiguration().getString("logging.dir");
 +      if (null != mDir) {
 +        File dir = new File(mDir);
 +        if (!dir.isDirectory())
 +          if (!dir.mkdir()) 
 +            log.warn("Could not create log directory: " + dir);
 +        logDir = dir;
 +        // Create new log file
 +        startNewLog();
 +      }
 +      currentlyLogging = true;
 +    }
 +  }
 +  
 +  private void startNewLog() throws IOException {
 +    if (null != logWriter) {
 +      logWriter.flush();
 +      logWriter.close();
 +    }
 +    logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
 +    if (!logFile.exists()) {
 +      if (!logFile.createNewFile()) {
 +        log.error("Unable to create new log file");
 +        currentlyLogging = false;
 +        return;
 +      }
 +    }
-     logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
++    logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), UTF_8);
 +  }
 +  
 +  private void writeToLog(String name) throws IOException {
 +    if (null == logWriter)
 +      return;
 +    // Increment the date if we have to
 +    Date now = new Date();
 +    if (!DateUtils.isSameDay(today, now)) {
 +      today = now;
 +      startNewLog();
 +    }
 +    logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
 +  }
 +  
 +  public void add(String name, long time) {
 +    if (isEnabled()) {
 +      registry.get(name).incCount();
 +      registry.get(name).addAvg(time);
 +      registry.get(name).addMin(time);
 +      registry.get(name).addMax(time);
 +      // If we are not currently logging and should be, then initialize
 +      if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +        try {
 +          setupLogging();
 +        } catch (IOException ioe) {
 +          log.error("Error setting up log", ioe);
 +        }
 +      } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
 +        // if we are currently logging and shouldn't be, then close logs
 +        try {
 +          logWriter.flush();
 +          logWriter.close();
 +          logWriter = null;
 +          logFile = null;
 +        } catch (Exception e) {
 +          log.error("Error stopping metrics logging", e);
 +        }
 +        currentlyLogging = false;
 +      }
 +      if (currentlyLogging) {
 +        try {
 +          writeToLog(name);
 +        } catch (IOException ioe) {
 +          log.error("Error writing to metrics log", ioe);
 +        }
 +      }
 +    }
 +  }
 +  
 +  public boolean isEnabled() {
 +    return config.isEnabled();
 +  }
 +  
 +  protected abstract ObjectName getObjectName();
 +  
 +  protected abstract String getMetricsPrefix();
 +  
 +  @Override
 +  protected void finalize() {
 +    if (null != logWriter) {
 +      try {
 +        logWriter.close();
 +      } catch (Exception e) {
 +        // do nothing
 +      } finally {
 +        logWriter = null;
 +      }
 +    }
 +    logFile = null;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
index 0a5341a,0000000..4e909c0
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
@@@ -1,187 -1,0 +1,189 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.monitor;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.net.ServerSocket;
 +import java.net.Socket;
 +import java.util.ArrayList;
 +import java.util.LinkedHashMap;
 +import java.util.List;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.LogManager;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.net.SocketNode;
 +import org.apache.log4j.spi.LoggingEvent;
 +import org.apache.zookeeper.KeeperException;
 +
 +/**
 + * Hijack log4j and capture log events for display.
 + * 
 + */
 +public class LogService extends org.apache.log4j.AppenderSkeleton {
 +
 +  private static final Logger log = Logger.getLogger(LogService.class);
 +
 +  /**
 +   * Read logging events forward to us over the net.
 +   * 
 +   */
 +  static class SocketServer implements Runnable {
 +    private ServerSocket server = null;
 +
 +    public SocketServer(int port) {
 +      try {
 +        server = new ServerSocket(port);
 +      } catch (IOException io) {
 +        throw new RuntimeException(io);
 +      }
 +    }
 +
 +    public int getLocalPort() {
 +      return server.getLocalPort();
 +    }
 +
 +    @Override
 +    public void run() {
 +      try {
 +        while (true) {
 +          log.debug("Waiting for log message senders");
 +          Socket socket = server.accept();
 +          log.debug("Got a new connection");
 +          Thread t = new Daemon(new SocketNode(socket, LogManager.getLoggerRepository()));
 +          t.start();
 +        }
 +      } catch (IOException io) {
 +        log.error(io, io);
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
 +   * 
 +   * @param conf
 +   *          configuration for the instance
 +   * @param instanceId
 +   *          instanceId for the instance
 +   * @param hostAddress
 +   *          Address that monitor process is bound to
 +   */
 +  public static void startLogListener(AccumuloConfiguration conf, String instanceId, String hostAddress) {
 +    try {
 +      SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));
 +
 +      // getLocalPort will return the actual ephemeral port used when '0' was provided.
 +      String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
 +
 +      log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);
 +
 +      final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
 +      final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +      // Delete before we try to re-create in case the previous session hasn't yet expired
 +      try {
 +        zoo.delete(path, -1);
 +      } catch (KeeperException e) {
 +        // We don't care if the node is already gone
 +        if (!KeeperException.Code.NONODE.equals(e.code())) {
 +          throw e;
 +        }
 +      }
 +
-       zoo.putEphemeralData(path, logForwardingAddr.getBytes(Constants.UTF8));
++      zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
 +
 +      new Daemon(server).start();
 +    } catch (Throwable t) {
 +      log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
 +    }
 +  }
 +
 +  static private LogService instance = null;
 +
 +  synchronized public static LogService getInstance() {
 +    if (instance == null)
 +      return new LogService();
 +    return instance;
 +  }
 +
 +  private static final int MAX_LOGS = 50;
 +
 +  private LinkedHashMap<String,DedupedLogEvent> events = new LinkedHashMap<String,DedupedLogEvent>(MAX_LOGS + 1, (float) .75, true) {
 +
 +    private static final long serialVersionUID = 1L;
 +
 +    @Override
 +    @SuppressWarnings("rawtypes")
 +    protected boolean removeEldestEntry(Map.Entry eldest) {
 +      return size() > MAX_LOGS;
 +    }
 +  };
 +
 +  public LogService() {
 +    synchronized (LogService.class) {
 +      instance = this;
 +    }
 +  }
 +
 +  @Override
 +  synchronized protected void append(LoggingEvent ev) {
 +    Object application = ev.getMDC("application");
 +    if (application == null || application.toString().isEmpty())
 +      return;
 +
 +    DedupedLogEvent dev = new DedupedLogEvent(ev);
 +
 +    // if event is present, increase the count
 +    if (events.containsKey(dev.toString())) {
 +      DedupedLogEvent oldDev = events.remove(dev.toString());
 +      dev.setCount(oldDev.getCount() + 1);
 +    }
 +    events.put(dev.toString(), dev);
 +  }
 +
 +  @Override
 +  public void close() {
 +    events = null;
 +  }
 +
 +  @Override
 +  public synchronized void doAppend(LoggingEvent event) {
 +    super.doAppend(event);
 +  }
 +
 +  @Override
 +  public boolean requiresLayout() {
 +    return false;
 +  }
 +
 +  synchronized public List<DedupedLogEvent> getEvents() {
 +    return new ArrayList<DedupedLogEvent>(events.values());
 +  }
 +
 +  synchronized public void clear() {
 +    events.clear();
 +  }
 +}


[33/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 03635cc,0000000..9cd02eb
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -1,1051 -1,0 +1,1053 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.BatchWriterImpl;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Writer;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.util.FastFormat;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +/**
 + * provides a reference to the metadata table for updates by tablet servers
 + */
 +public class MetadataTableUtil {
 +
 +  private static final Text EMPTY_TEXT = new Text();
 +  private static final byte[] EMPTY_BYTES = new byte[0];
 +  private static Map<Credentials,Writer> root_tables = new HashMap<Credentials,Writer>();
 +  private static Map<Credentials,Writer> metadata_tables = new HashMap<Credentials,Writer>();
 +  private static final Logger log = Logger.getLogger(MetadataTableUtil.class);
 +
 +  private MetadataTableUtil() {}
 +
 +  public synchronized static Writer getMetadataTable(Credentials credentials) {
 +    Writer metadataTable = metadata_tables.get(credentials);
 +    if (metadataTable == null) {
 +      metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID);
 +      metadata_tables.put(credentials, metadataTable);
 +    }
 +    return metadataTable;
 +  }
 +
 +  private synchronized static Writer getRootTable(Credentials credentials) {
 +    Writer rootTable = root_tables.get(credentials);
 +    if (rootTable == null) {
 +      rootTable = new Writer(HdfsZooInstance.getInstance(), credentials, RootTable.ID);
 +      root_tables.put(credentials, rootTable);
 +    }
 +    return rootTable;
 +  }
 +
 +  private static void putLockID(ZooLock zooLock, Mutation m) {
 +    TabletsSection.ServerColumnFamily.LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/")
-         .getBytes(Constants.UTF8)));
++        .getBytes(UTF_8)));
 +  }
 +
 +  private static void update(Credentials credentials, Mutation m, KeyExtent extent) {
 +    update(credentials, null, m, extent);
 +  }
 +
 +  public static void update(Credentials credentials, ZooLock zooLock, Mutation m, KeyExtent extent) {
 +    Writer t = extent.isMeta() ? getRootTable(credentials) : getMetadataTable(credentials);
 +    update(t, credentials, zooLock, m);
 +  }
 +
 +  public static void update(Writer t, Credentials credentials, ZooLock zooLock, Mutation m) {
 +    if (zooLock != null)
 +      putLockID(zooLock, m);
 +    while (true) {
 +      try {
 +        t.update(m);
 +        return;
 +      } catch (AccumuloException e) {
 +        log.error(e, e);
 +      } catch (AccumuloSecurityException e) {
 +        log.error(e, e);
 +      } catch (ConstraintViolationException e) {
 +        log.error(e, e);
 +        // retrying when a CVE occurs is probably futile and can cause problems, see ACCUMULO-3096
 +        throw new RuntimeException(e);
 +      } catch (TableNotFoundException e) {
 +        log.error(e, e);
 +      }
 +      UtilWaitThread.sleep(1000);
 +    }
 +  }
 +
 +  public static void updateTabletFlushID(KeyExtent extent, long flushID, Credentials credentials, ZooLock zooLock) {
 +    if (!extent.isRootTablet()) {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
-       TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(Constants.UTF8)));
++      TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(UTF_8)));
 +      update(credentials, zooLock, m, extent);
 +    }
 +  }
 +
 +  public static void updateTabletCompactID(KeyExtent extent, long compactID, Credentials credentials, ZooLock zooLock) {
 +    if (!extent.isRootTablet()) {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
-       TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(Constants.UTF8)));
++      TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(UTF_8)));
 +      update(credentials, zooLock, m, extent);
 +    }
 +  }
 +
 +  public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, Credentials credentials, ZooLock zooLock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
-     byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
++    byte[] tidBytes = Long.toString(tid).getBytes(UTF_8);
 +
 +    for (Entry<FileRef,DataFileValue> entry : estSizes.entrySet()) {
 +      Text file = entry.getKey().meta();
 +      m.put(DataFileColumnFamily.NAME, file, new Value(entry.getValue().encode()));
 +      m.put(TabletsSection.BulkFileColumnFamily.NAME, file, new Value(tidBytes));
 +    }
-     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
++    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
 +    update(credentials, zooLock, m, extent);
 +  }
 +
 +  public static void updateTabletDir(KeyExtent extent, String newDir, Credentials creds, ZooLock lock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
-     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
 +    update(creds, lock, m, extent);
 +  }
 +
 +  public static void addTablet(KeyExtent extent, String path, Credentials credentials, char timeType, ZooLock lock) {
 +    Mutation m = extent.getPrevRowUpdateMutation();
 +
-     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
-     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
++    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
++    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(UTF_8)));
 +
 +    update(credentials, lock, m, extent);
 +  }
 +
 +  public static void updateTabletPrevEndRow(KeyExtent extent, Credentials credentials) {
 +    Mutation m = extent.getPrevRowUpdateMutation(); //
 +    update(credentials, m, extent);
 +  }
 +
 +  public static void updateTabletVolumes(KeyExtent extent, List<LogEntry> logsToRemove, List<LogEntry> logsToAdd, List<FileRef> filesToRemove,
 +      SortedMap<FileRef,DataFileValue> filesToAdd, String newDir, ZooLock zooLock, Credentials credentials) {
 +
 +    if (extent.isRootTablet()) {
 +      if (newDir != null)
 +        throw new IllegalArgumentException("newDir not expected for " + extent);
 +
 +      if (filesToRemove.size() != 0 || filesToAdd.size() != 0)
 +        throw new IllegalArgumentException("files not expected for " + extent);
 +
 +      // add before removing in case of process death
 +      for (LogEntry logEntry : logsToAdd)
 +        addLogEntry(credentials, logEntry, zooLock);
 +
 +      removeUnusedWALEntries(extent, logsToRemove, zooLock);
 +    } else {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
 +
 +      for (LogEntry logEntry : logsToRemove)
 +        m.putDelete(logEntry.getColumnFamily(), logEntry.getColumnQualifier());
 +
 +      for (LogEntry logEntry : logsToAdd)
 +        m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
 +
 +      for (FileRef fileRef : filesToRemove)
 +        m.putDelete(DataFileColumnFamily.NAME, fileRef.meta());
 +
 +      for (Entry<FileRef,DataFileValue> entry : filesToAdd.entrySet())
 +        m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
 +
 +      if (newDir != null)
-         ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++        ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
 +
 +      update(credentials, m, extent);
 +    }
 +  }
 +
 +  public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, Credentials credentials) throws IOException {
 +    TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
 +
 +    Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
 +    mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    Text row = extent.getMetadataEntry();
 +    VolumeManager fs = VolumeManagerImpl.get();
 +
 +    Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text(""));
 +    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
 +
 +    mdScanner.setRange(new Range(new Key(row), endKey));
 +    for (Entry<Key,Value> entry : mdScanner) {
 +
 +      if (!entry.getKey().getRow().equals(row))
 +        break;
 +      DataFileValue dfv = new DataFileValue(entry.getValue().get());
 +      sizes.put(new FileRef(fs, entry.getKey()), dfv);
 +    }
 +
 +    return sizes;
 +  }
 +
 +  public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, Credentials credentials, ZooLock zooLock) {
 +    KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
 +    Mutation m = ke.getPrevRowUpdateMutation();
 +    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
 +    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
 +    update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
 +  }
 +
 +  public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, Credentials credentials, ZooLock zooLock) {
 +    Mutation m = extent.getPrevRowUpdateMutation(); //
 +
-     TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
++    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(UTF_8)));
 +
 +    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
 +    ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
 +    update(credentials, zooLock, m, extent);
 +  }
 +
 +  public static void finishSplit(Text metadataEntry, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
 +      ZooLock zooLock) {
 +    Mutation m = new Mutation(metadataEntry);
 +    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
 +    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
 +    ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
 +
 +    for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
 +      m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
 +    }
 +
 +    for (FileRef pathToRemove : highDatafilesToRemove) {
 +      m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta());
 +    }
 +
 +    update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
 +  }
 +
 +  public static void finishSplit(KeyExtent extent, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
 +      ZooLock zooLock) {
 +    finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
 +  }
 +
 +  public static void addDeleteEntries(KeyExtent extent, Set<FileRef> datafilesToDelete, Credentials credentials) throws IOException {
 +
 +    String tableId = extent.getTableId().toString();
 +
 +    // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
 +    for (FileRef pathToRemove : datafilesToDelete) {
 +      update(credentials, createDeleteMutation(tableId, pathToRemove.path().toString()), extent);
 +    }
 +  }
 +
 +  public static void addDeleteEntry(String tableId, String path) throws IOException {
 +    update(SystemCredentials.get(), createDeleteMutation(tableId, path), new KeyExtent(new Text(tableId), null, null));
 +  }
 +
 +  public static Mutation createDeleteMutation(String tableId, String pathToRemove) throws IOException {
 +    Path path = VolumeManagerImpl.get().getFullPath(tableId, pathToRemove);
 +    Mutation delFlag = new Mutation(new Text(MetadataSchema.DeletesSection.getRowPrefix() + path.toString()));
 +    delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
 +    return delFlag;
 +  }
 +
 +  public static void removeScanFiles(KeyExtent extent, Set<FileRef> scanFiles, Credentials credentials, ZooLock zooLock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
 +
 +    for (FileRef pathToRemove : scanFiles)
 +      m.putDelete(ScanFileColumnFamily.NAME, pathToRemove.meta());
 +
 +    update(credentials, zooLock, m, extent);
 +  }
 +
 +  public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<FileRef,FileUtil.FileInfo> firstAndLastRows,
 +      SortedMap<FileRef,DataFileValue> datafiles, SortedMap<FileRef,DataFileValue> lowDatafileSizes, SortedMap<FileRef,DataFileValue> highDatafileSizes,
 +      List<FileRef> highDatafilesToRemove) {
 +
 +    for (Entry<FileRef,DataFileValue> entry : datafiles.entrySet()) {
 +
 +      Text firstRow = null;
 +      Text lastRow = null;
 +
 +      boolean rowsKnown = false;
 +
 +      FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
 +
 +      if (mfi != null) {
 +        firstRow = mfi.getFirstRow();
 +        lastRow = mfi.getLastRow();
 +        rowsKnown = true;
 +      }
 +
 +      if (rowsKnown && firstRow.compareTo(midRow) > 0) {
 +        // only in high
 +        long highSize = entry.getValue().getSize();
 +        long highEntries = entry.getValue().getNumEntries();
 +        highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
 +      } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
 +        // only in low
 +        long lowSize = entry.getValue().getSize();
 +        long lowEntries = entry.getValue().getNumEntries();
 +        lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
 +
 +        highDatafilesToRemove.add(entry.getKey());
 +      } else {
 +        long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
 +        long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
 +        lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
 +
 +        long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
 +        long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
 +        highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
 +      }
 +    }
 +  }
 +
 +  public static void deleteTable(String tableId, boolean insertDeletes, Credentials credentials, ZooLock lock) throws AccumuloException, IOException {
 +    Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
 +    Text tableIdText = new Text(tableId);
 +    BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000)
 +        .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
 +
 +    // scan metadata for our table and delete everything we find
 +    Mutation m = null;
 +    ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
 +
 +    // insert deletes before deleting data from metadata... this makes the code fault tolerant
 +    if (insertDeletes) {
 +
 +      ms.fetchColumnFamily(DataFileColumnFamily.NAME);
 +      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms);
 +
 +      for (Entry<Key,Value> cell : ms) {
 +        Key key = cell.getKey();
 +
 +        if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +          FileRef ref = new FileRef(VolumeManagerImpl.get(), key);
 +          bw.addMutation(createDeleteMutation(tableId, ref.meta().toString()));
 +        }
 +
 +        if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
 +          bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
 +        }
 +      }
 +
 +      bw.flush();
 +
 +      ms.clearColumns();
 +    }
 +
 +    for (Entry<Key,Value> cell : ms) {
 +      Key key = cell.getKey();
 +
 +      if (m == null) {
 +        m = new Mutation(key.getRow());
 +        if (lock != null)
 +          putLockID(lock, m);
 +      }
 +
 +      if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
 +        bw.addMutation(m);
 +        m = new Mutation(key.getRow());
 +        if (lock != null)
 +          putLockID(lock, m);
 +      }
 +      m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
 +    }
 +
 +    if (m != null)
 +      bw.addMutation(m);
 +
 +    bw.close();
 +  }
 +
 +  static String getZookeeperLogLocation() {
 +    return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_WALOGS;
 +  }
 +
 +  public static void addLogEntry(Credentials credentials, LogEntry entry, ZooLock zooLock) {
 +    if (entry.extent.isRootTablet()) {
 +      String root = getZookeeperLogLocation();
 +      while (true) {
 +        try {
 +          IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +          if (zoo.isLockHeld(zooLock.getLockID())) {
 +            String[] parts = entry.filename.split("/");
 +            String uniqueId = parts[parts.length - 1];
 +            zoo.putPersistentData(root + "/" + uniqueId, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
 +          }
 +          break;
 +        } catch (KeeperException e) {
 +          log.error(e, e);
 +        } catch (InterruptedException e) {
 +          log.error(e, e);
 +        } catch (IOException e) {
 +          log.error(e, e);
 +        }
 +        UtilWaitThread.sleep(1000);
 +      }
 +    } else {
 +      Mutation m = new Mutation(entry.getRow());
 +      m.put(entry.getColumnFamily(), entry.getColumnQualifier(), entry.getValue());
 +      update(credentials, zooLock, m, entry.extent);
 +    }
 +  }
 +
 +  public static void setRootTabletDir(String dir) throws IOException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
 +    try {
-       zoo.putPersistentData(zpath, dir.getBytes(Constants.UTF8), -1, NodeExistsPolicy.OVERWRITE);
++      zoo.putPersistentData(zpath, dir.getBytes(UTF_8), -1, NodeExistsPolicy.OVERWRITE);
 +    } catch (KeeperException e) {
 +      throw new IOException(e);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  public static String getRootTabletDir() throws IOException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
 +    try {
-       return new String(zoo.getData(zpath, null), Constants.UTF8);
++      return new String(zoo.getData(zpath, null), UTF_8);
 +    } catch (KeeperException e) {
 +      throw new IOException(e);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  public static Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> getFileAndLogEntries(Credentials credentials, KeyExtent extent) throws KeeperException,
 +      InterruptedException, IOException {
 +    ArrayList<LogEntry> result = new ArrayList<LogEntry>();
 +    TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
 +
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    if (extent.isRootTablet()) {
 +      getRootLogEntries(result);
 +      Path rootDir = new Path(getRootTabletDir());
 +      FileStatus[] files = fs.listStatus(rootDir);
 +      for (FileStatus fileStatus : files) {
 +        if (fileStatus.getPath().toString().endsWith("_tmp")) {
 +          continue;
 +        }
 +        DataFileValue dfv = new DataFileValue(0, 0);
 +        sizes.put(new FileRef(fileStatus.getPath().toString(), fileStatus.getPath()), dfv);
 +      }
 +
 +    } else {
 +      String systemTableToCheck = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
 +      Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, systemTableToCheck, Authorizations.EMPTY);
 +      scanner.fetchColumnFamily(LogColumnFamily.NAME);
 +      scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +      scanner.setRange(extent.toMetadataRange());
 +
 +      for (Entry<Key,Value> entry : scanner) {
 +        if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
 +          throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
 +        }
 +
 +        if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
 +          result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
 +        } else if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +          DataFileValue dfv = new DataFileValue(entry.getValue().get());
 +          sizes.put(new FileRef(fs, entry.getKey()), dfv);
 +        } else {
 +          throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
 +        }
 +      }
 +    }
 +
 +    return new Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>>(result, sizes);
 +  }
 +
 +  public static List<LogEntry> getLogEntries(Credentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
 +    log.info("Scanning logging entries for " + extent);
 +    ArrayList<LogEntry> result = new ArrayList<LogEntry>();
 +    if (extent.equals(RootTable.EXTENT)) {
 +      log.info("Getting logs for root tablet from zookeeper");
 +      getRootLogEntries(result);
 +    } else {
 +      log.info("Scanning metadata for logs used for tablet " + extent);
 +      Scanner scanner = getTabletLogScanner(credentials, extent);
 +      Text pattern = extent.getMetadataEntry();
 +      for (Entry<Key,Value> entry : scanner) {
 +        Text row = entry.getKey().getRow();
 +        if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
 +          if (row.equals(pattern)) {
 +            result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
 +          }
 +        }
 +      }
 +    }
 +
 +    Collections.sort(result, new Comparator<LogEntry>() {
 +      @Override
 +      public int compare(LogEntry o1, LogEntry o2) {
 +        long diff = o1.timestamp - o2.timestamp;
 +        if (diff < 0)
 +          return -1;
 +        if (diff > 0)
 +          return 1;
 +        return 0;
 +      }
 +    });
 +    log.info("Returning logs " + result + " for extent " + extent);
 +    return result;
 +  }
 +
 +  static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    String root = getZookeeperLogLocation();
 +    // there's a little race between getting the children and fetching
 +    // the data. The log can be removed in between.
 +    while (true) {
 +      result.clear();
 +      for (String child : zoo.getChildren(root)) {
 +        LogEntry e = new LogEntry();
 +        try {
 +          e.fromBytes(zoo.getData(root + "/" + child, null));
 +          // upgrade from !0;!0<< -> +r<<
 +          e.extent = RootTable.EXTENT;
 +          result.add(e);
 +        } catch (KeeperException.NoNodeException ex) {
 +          continue;
 +        }
 +      }
 +      break;
 +    }
 +  }
 +
 +  private static Scanner getTabletLogScanner(Credentials credentials, KeyExtent extent) {
 +    String tableId = MetadataTable.ID;
 +    if (extent.isMeta())
 +      tableId = RootTable.ID;
 +    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, tableId, Authorizations.EMPTY);
 +    scanner.fetchColumnFamily(LogColumnFamily.NAME);
 +    Text start = extent.getMetadataEntry();
 +    Key endKey = new Key(start, LogColumnFamily.NAME);
 +    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
 +    scanner.setRange(new Range(new Key(start), endKey));
 +    return scanner;
 +  }
 +
 +  private static class LogEntryIterator implements Iterator<LogEntry> {
 +
 +    Iterator<LogEntry> zookeeperEntries = null;
 +    Iterator<LogEntry> rootTableEntries = null;
 +    Iterator<Entry<Key,Value>> metadataEntries = null;
 +
 +    LogEntryIterator(Credentials creds) throws IOException, KeeperException, InterruptedException {
 +      zookeeperEntries = getLogEntries(creds, RootTable.EXTENT).iterator();
 +      rootTableEntries = getLogEntries(creds, new KeyExtent(new Text(MetadataTable.ID), null, null)).iterator();
 +      try {
 +        Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken())
 +            .createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +        log.info("Setting range to " + MetadataSchema.TabletsSection.getRange());
 +        scanner.setRange(MetadataSchema.TabletsSection.getRange());
 +        scanner.fetchColumnFamily(LogColumnFamily.NAME);
 +        metadataEntries = scanner.iterator();
 +      } catch (Exception ex) {
 +        throw new IOException(ex);
 +      }
 +    }
 +
 +    @Override
 +    public boolean hasNext() {
 +      return zookeeperEntries.hasNext() || rootTableEntries.hasNext() || metadataEntries.hasNext();
 +    }
 +
 +    @Override
 +    public LogEntry next() {
 +      if (zookeeperEntries.hasNext()) {
 +        return zookeeperEntries.next();
 +      }
 +      if (rootTableEntries.hasNext()) {
 +        return rootTableEntries.next();
 +      }
 +      Entry<Key,Value> entry = metadataEntries.next();
 +      return LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
 +    }
 +
 +    @Override
 +    public void remove() {
 +      throw new UnsupportedOperationException();
 +    }
 +  }
 +
 +  public static Iterator<LogEntry> getLogEntries(Credentials creds) throws IOException, KeeperException, InterruptedException {
 +    return new LogEntryIterator(creds);
 +  }
 +
 +  public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
 +    if (extent.isRootTablet()) {
 +      for (LogEntry entry : logEntries) {
 +        String root = getZookeeperLogLocation();
 +        while (true) {
 +          try {
 +            IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +            if (zoo.isLockHeld(zooLock.getLockID())) {
 +              String parts[] = entry.filename.split("/");
 +              zoo.recursiveDelete(root + "/" + parts[parts.length - 1], NodeMissingPolicy.SKIP);
 +            }
 +            break;
 +          } catch (Exception e) {
 +            log.error(e, e);
 +          }
 +          UtilWaitThread.sleep(1000);
 +        }
 +      }
 +    } else {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
 +      for (LogEntry entry : logEntries) {
 +        m.putDelete(LogColumnFamily.NAME, new Text(entry.getName()));
 +      }
 +      update(SystemCredentials.get(), zooLock, m, extent);
 +    }
 +  }
 +
 +  private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
 +    for (Entry<Key,Value> entry : tablet.entrySet()) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        String cf = entry.getKey().getColumnQualifier().toString();
 +        if (srcTableId != null && !cf.startsWith("../") && !cf.contains(":")) {
 +          cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
 +        }
 +        files.add(cf);
 +      }
 +    }
 +  }
 +
 +  private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
 +
 +    KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
 +    Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
 +
 +    for (Entry<Key,Value> entry : tablet.entrySet()) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        String cf = entry.getKey().getColumnQualifier().toString();
 +        if (!cf.startsWith("../") && !cf.contains(":"))
 +          cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
 +        m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
 +      } else if (entry.getKey().getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
 +        m.put(TabletsSection.LastLocationColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
 +      } else if (entry.getKey().getColumnFamily().equals(TabletsSection.LastLocationColumnFamily.NAME)) {
 +        // skip
 +      } else {
 +        m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
 +      }
 +    }
 +    return m;
 +  }
 +
 +  private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
 +    String tableName = MetadataTable.NAME;
 +    if (tableId.equals(MetadataTable.ID))
 +      tableName = RootTable.NAME;
 +    Scanner mscanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    mscanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +    mscanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
 +    mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
 +    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(mscanner);
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(mscanner);
 +    return mscanner;
 +  }
 +
 +  static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
 +    TabletIterator ti;
 +    if (srcTableId.equals(MetadataTable.ID))
 +      ti = new TabletIterator(createCloneScanner(srcTableId, conn), new Range(), true, true);
 +    else
 +      ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
 +
 +    if (!ti.hasNext())
 +      throw new RuntimeException(" table deleted during clone?  srcTableId = " + srcTableId);
 +
 +    while (ti.hasNext())
 +      bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
 +
 +    bw.flush();
 +  }
 +
 +  private static int compareEndRows(Text endRow1, Text endRow2) {
 +    return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
 +  }
 +
 +  static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
 +    TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
 +        true);
 +    TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
 +
 +    if (!cloneIter.hasNext() || !srcIter.hasNext())
 +      throw new RuntimeException(" table deleted during clone?  srcTableId = " + srcTableId + " tableId=" + tableId);
 +
 +    int rewrites = 0;
 +
 +    while (cloneIter.hasNext()) {
 +      Map<Key,Value> cloneTablet = cloneIter.next();
 +      Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
 +      HashSet<String> cloneFiles = new HashSet<String>();
 +
 +      boolean cloneSuccessful = false;
 +      for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
 +        if (entry.getKey().getColumnFamily().equals(ClonedColumnFamily.NAME)) {
 +          cloneSuccessful = true;
 +          break;
 +        }
 +      }
 +
 +      if (!cloneSuccessful)
 +        getFiles(cloneFiles, cloneTablet, null);
 +
 +      List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
 +      Map<Key,Value> srcTablet = srcIter.next();
 +      srcTablets.add(srcTablet);
 +
 +      Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
 +
 +      int cmp = compareEndRows(cloneEndRow, srcEndRow);
 +      if (cmp < 0)
 +        throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
 +
 +      HashSet<String> srcFiles = new HashSet<String>();
 +      if (!cloneSuccessful)
 +        getFiles(srcFiles, srcTablet, srcTableId);
 +
 +      while (cmp > 0) {
 +        srcTablet = srcIter.next();
 +        srcTablets.add(srcTablet);
 +        srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
 +        cmp = compareEndRows(cloneEndRow, srcEndRow);
 +        if (cmp < 0)
 +          throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
 +
 +        if (!cloneSuccessful)
 +          getFiles(srcFiles, srcTablet, srcTableId);
 +      }
 +
 +      if (cloneSuccessful)
 +        continue;
 +
 +      if (!srcFiles.containsAll(cloneFiles)) {
 +        // delete existing cloned tablet entry
 +        Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
 +
 +        for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
 +          Key k = entry.getKey();
 +          m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
 +        }
 +
 +        bw.addMutation(m);
 +
 +        for (Map<Key,Value> st : srcTablets)
 +          bw.addMutation(createCloneMutation(srcTableId, tableId, st));
 +
 +        rewrites++;
 +      } else {
 +        // write out marker that this tablet was successfully cloned
 +        Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
-         m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
++        m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(UTF_8)));
 +        bw.addMutation(m);
 +      }
 +    }
 +
 +    bw.flush();
 +    return rewrites;
 +  }
 +
 +  public static void cloneTable(Instance instance, String srcTableId, String tableId, VolumeManager volumeManager) throws Exception {
 +
 +    Connector conn = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
 +    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +
 +    while (true) {
 +
 +      try {
 +        initializeClone(srcTableId, tableId, conn, bw);
 +
 +        // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
 +
 +        while (true) {
 +          int rewrites = checkClone(srcTableId, tableId, conn, bw);
 +
 +          if (rewrites == 0)
 +            break;
 +        }
 +
 +        bw.flush();
 +        break;
 +
 +      } catch (TabletIterator.TabletDeletedException tde) {
 +        // tablets were merged in the src table
 +        bw.flush();
 +
 +        // delete what we have cloned and try again
 +        deleteTable(tableId, false, SystemCredentials.get(), null);
 +
 +        log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
 +
 +        UtilWaitThread.sleep(100);
 +      }
 +    }
 +
 +    // delete the clone markers and create directory entries
 +    Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
 +
 +    int dirCount = 0;
 +
 +    for (Entry<Key,Value> entry : mscanner) {
 +      Key k = entry.getKey();
 +      Mutation m = new Mutation(k.getRow());
 +      m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
 +      String dir = volumeManager.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId
 +          + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
-       TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(Constants.UTF8)));
++      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8)));
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +
 +  }
 +
 +  public static void chopped(KeyExtent extent, ZooLock zooLock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
-     ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
++    ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(UTF_8)));
 +    update(SystemCredentials.get(), zooLock, m, extent);
 +  }
 +
 +  public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
 +    Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +    for (Entry<Key,Value> entry : mscanner) {
 +      log.debug("Looking at entry " + entry + " with tid " + tid);
 +      if (Long.parseLong(entry.getValue().toString()) == tid) {
 +        log.debug("deleting entry " + entry);
 +        Mutation m = new Mutation(entry.getKey().getRow());
 +        m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
 +        bw.addMutation(m);
 +      }
 +    }
 +    bw.close();
 +  }
 +
 +  public static List<FileRef> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) throws IOException {
 +    List<FileRef> result = new ArrayList<FileRef>();
 +    try {
 +      VolumeManager fs = VolumeManagerImpl.get();
 +      Scanner mscanner = new IsolatedScanner(conn.createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY));
 +      mscanner.setRange(extent.toMetadataRange());
 +      mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +      for (Entry<Key,Value> entry : mscanner) {
 +        if (Long.parseLong(entry.getValue().toString()) == tid) {
 +          result.add(new FileRef(fs, entry.getKey()));
 +        }
 +      }
 +      return result;
 +    } catch (TableNotFoundException ex) {
 +      // unlikely
 +      throw new RuntimeException("Onos! teh metadata table has vanished!!");
 +    }
 +  }
 +
 +  public static Map<FileRef,Long> getBulkFilesLoaded(Credentials credentials, KeyExtent extent) throws IOException {
 +    Text metadataRow = extent.getMetadataEntry();
 +    Map<FileRef,Long> ret = new HashMap<FileRef,Long>();
 +
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, extent.isMeta() ? RootTable.ID : MetadataTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(new Range(metadataRow));
 +    scanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +    for (Entry<Key,Value> entry : scanner) {
 +      Long tid = Long.parseLong(entry.getValue().toString());
 +      ret.put(new FileRef(fs, entry.getKey()), tid);
 +    }
 +    return ret;
 +  }
 +
 +  public static void addBulkLoadInProgressFlag(String path) {
 +
 +    Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
 +    m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
 +
 +    // new KeyExtent is only added to force update to write to the metadata table, not the root table
 +    // because bulk loads aren't supported to the metadata table
 +    update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
 +  }
 +
 +  public static void removeBulkLoadInProgressFlag(String path) {
 +
 +    Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
 +    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
 +
 +    // new KeyExtent is only added to force update to write to the metadata table, not the root table
 +    // because bulk loads aren't supported to the metadata table
 +    update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
 +  }
 +
 +  /**
 +   * During an upgrade we need to move deletion requests for files under the !METADATA table to the root tablet.
 +   */
 +  public static void moveMetaDeleteMarkers(Instance instance, Credentials creds) {
 +    String oldDeletesPrefix = "!!~del";
 +    Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false);
 +
 +    // move old delete markers to new location, to standardize table schema between all metadata tables
 +    Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(oldDeletesRange);
 +    for (Entry<Key,Value> entry : scanner) {
 +      String row = entry.getKey().getRow().toString();
 +      if (row.startsWith(oldDeletesPrefix)) {
 +        moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row, oldDeletesPrefix);
 +      } else {
 +        break;
 +      }
 +    }
 +
 +  }
 +
 +  public static void moveMetaDeleteMarkersFrom14(Instance instance, Credentials creds) {
 +    // new KeyExtent is only added to force update to write to the metadata table, not the root table
 +    KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"), null, null);
 +
 +    // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
 +    Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(MetadataSchema.DeletesSection.getRange());
 +    for (Entry<Key,Value> entry : scanner) {
 +      String row = entry.getKey().getRow().toString();
 +      if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" + MetadataTable.ID)) {
 +        moveDeleteEntry(creds, notMetadata, entry, row, MetadataSchema.DeletesSection.getRowPrefix());
 +      } else {
 +        break;
 +      }
 +    }
 +  }
 +
 +  private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent, Entry<Key,Value> entry, String rowID, String prefix) {
 +    String filename = rowID.substring(prefix.length());
 +
 +    // add the new entry first
 +    log.info("Moving " + filename + " marker in " + RootTable.NAME);
 +    Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
 +    m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
 +    update(creds, m, RootTable.EXTENT);
 +
 +    // then remove the old entry
 +    m = new Mutation(entry.getKey().getRow());
 +    m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
 +    update(creds, m, oldExtent);
 +  }
 +
 +  public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
 +    TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
 +
 +    HashSet<ColumnFQ> colSet = null;
 +    if (columns != null) {
 +      colSet = new HashSet<ColumnFQ>(columns);
 +    }
 +
 +    for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
 +
 +      if (columns != null && !colSet.contains(new ColumnFQ(entry.getKey()))) {
 +        continue;
 +      }
 +
 +      Text row = entry.getKey().getRow();
 +
 +      SortedMap<ColumnFQ,Value> colVals = tabletEntries.get(row);
 +      if (colVals == null) {
 +        colVals = new TreeMap<ColumnFQ,Value>();
 +        tabletEntries.put(row, colVals);
 +      }
 +
 +      colVals.put(new ColumnFQ(entry.getKey()), entry.getValue());
 +    }
 +
 +    return tabletEntries;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index 4e5df9e,0000000..c0d6894
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class NamespacePropUtil {
 +  public static boolean setNamespaceProperty(String namespaceId, String property, String value) throws KeeperException, InterruptedException {
 +    if (!isPropertyValid(property, value))
 +      return false;
 +
 +    // create the zk node for per-namespace properties for this namespace if it doesn't already exist
 +    String zkNamespacePath = getPath(namespaceId);
 +    ZooReaderWriter.getInstance().putPersistentData(zkNamespacePath, new byte[0], NodeExistsPolicy.SKIP);
 +
 +    // create the zk node for this property and set it's data to the specified value
 +    String zPath = zkNamespacePath + "/" + property;
-     ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +
 +    return true;
 +  }
 +
 +  public static boolean isPropertyValid(String property, String value) {
 +    Property p = Property.getPropertyByKey(property);
 +    if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
 +      return false;
 +
 +    return true;
 +  }
 +
 +  public static void removeNamespaceProperty(String namespaceId, String property) throws InterruptedException, KeeperException {
 +    String zPath = getPath(namespaceId) + "/" + property;
 +    ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
 +  }
 +
 +  private static String getPath(String namespaceId) {
 +    return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZNAMESPACES + "/" + namespaceId + Constants.ZNAMESPACE_CONF;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index dd540f2,0000000..8924c85
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -1,131 -1,0 +1,131 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
- 
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 +
 +public class RandomizeVolumes {
 +  private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
 +
 +  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException {
 +    ClientOnRequiredTable opts = new ClientOnRequiredTable();
 +    opts.parseArgs(RandomizeVolumes.class.getName(), args);
 +    Connector c;
 +    if (opts.getToken() == null) {
 +      SystemCredentials creds = SystemCredentials.get();
 +      c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
 +    } else {
 +      c = opts.getConnector();
 +    }
 +    try {
 +      int status = randomize(c, opts.tableName);
 +      System.exit(status);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +      System.exit(4);
 +    }
 +  }
 +
 +  public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    VolumeManager vm = VolumeManagerImpl.get();
 +    if (vm.getVolumes().size() < 2) {
 +      log.error("There are not enough volumes configured");
 +      return 1;
 +    }
 +    String tableId = c.tableOperations().tableIdMap().get(tableName);
 +    if (null == tableId) {
 +      log.error("Could not determine the table ID for table " + tableName);
 +      return 2;
 +    }
 +    TableState tableState = TableManager.getInstance().getTableState(tableId);
 +    if (TableState.OFFLINE != tableState) {
 +      log.info("Taking " + tableName + " offline");
 +      c.tableOperations().offline(tableName, true);
 +      log.info(tableName + " offline");
 +    }
 +    log.info("Rewriting entries for " + tableName);
 +    Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    DIRECTORY_COLUMN.fetch(scanner);
 +    scanner.setRange(TabletsSection.getRange(tableId));
 +    BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      String oldLocation = entry.getValue().toString();
 +      String directory;
 +      if (oldLocation.contains(":")) {
 +        String[] parts = oldLocation.split(Path.SEPARATOR);
 +        String tableIdEntry = parts[parts.length - 2];
 +        if (!tableIdEntry.equals(tableId)) {
 +          log.error("Unexpected table id found: " + tableIdEntry + ", expected " + tableId + "; skipping");
 +          continue;
 +        }
 +        directory = parts[parts.length - 1];
 +      } else {
 +        directory = oldLocation.substring(Path.SEPARATOR.length());
 +      }
 +      Key key = entry.getKey();
 +      Mutation m = new Mutation(key.getRow());
 +      
 +      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
-       m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(Constants.UTF8)));
++      m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
 +      if (log.isTraceEnabled()) {
 +        log.trace("Replacing " + oldLocation + " with " + newLocation);
 +      }
 +      writer.addMutation(m);
 +      try {
 +        vm.mkdirs(new Path(newLocation));
 +      } catch (IOException ex) {
 +        // nevermind
 +      }
 +      count++;
 +    }
 +    writer.close();
 +    log.info("Updated " + count + " entries for table " + tableName);
 +    if (TableState.OFFLINE != tableState) {
 +      c.tableOperations().online(tableName, true);
 +      log.info("table " + tableName + " back online");
 +    }
 +    return 0;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index a08000e,0000000..a82fa14
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,124 -1,0 +1,125 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.FileInputStream;
 +import java.io.InputStream;
 +import java.util.Stack;
 +
 +import javax.xml.parsers.SAXParser;
 +import javax.xml.parsers.SAXParserFactory;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.xml.sax.Attributes;
 +import org.xml.sax.SAXException;
 +import org.xml.sax.helpers.DefaultHandler;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class RestoreZookeeper {
 +  
 +  private static class Restore extends DefaultHandler {
 +    IZooReaderWriter zk = null;
 +    Stack<String> cwd = new Stack<String>();
 +    boolean overwrite = false;
 +    
 +    Restore(IZooReaderWriter zk, boolean overwrite) {
 +      this.zk = zk;
 +      this.overwrite = overwrite;
 +    }
 +    
 +    @Override
 +    public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
 +      if ("node".equals(name)) {
 +        String child = attributes.getValue("name");
 +        if (child == null)
 +          throw new RuntimeException("name attribute not set");
 +        String encoding = attributes.getValue("encoding");
 +        String value = attributes.getValue("value");
 +        if (value == null)
 +          value = "";
 +        String path = cwd.lastElement() + "/" + child;
 +        create(path, value, encoding);
 +        cwd.push(path);
 +      } else if ("dump".equals(name)) {
 +        String root = attributes.getValue("root");
 +        if (root.equals("/"))
 +          cwd.push("");
 +        else
 +          cwd.push(root);
-         create(root, "", Constants.UTF8.name());
++        create(root, "", UTF_8.name());
 +      }
 +    }
 +    
 +    @Override
 +    public void endElement(String uri, String localName, String name) throws SAXException {
 +      cwd.pop();
 +    }
 +    
 +    // assume UTF-8 if not "base64"
 +    private void create(String path, String value, String encoding) {
-       byte[] data = value.getBytes(Constants.UTF8);
++      byte[] data = value.getBytes(UTF_8);
 +      if ("base64".equals(encoding))
 +        data = Base64.decodeBase64(data);
 +      try {
 +        try {
 +          zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
 +        } catch (KeeperException e) {
 +          if (e.code().equals(KeeperException.Code.NODEEXISTS))
 +            throw new RuntimeException(path + " exists.  Remove it first.");
 +          throw e;
 +        }
 +      } catch (Exception e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = {"-z", "--keepers"})
 +    String keepers = "localhost:2181";
 +    @Parameter(names = "--overwrite")
 +    boolean overwrite = false;
 +    @Parameter(names = "--file")
 +    String file;
 +  }
 +  
 +  public static void main(String[] args) throws Exception {
 +    Logger.getRootLogger().setLevel(Level.WARN);
 +    Opts opts = new Opts();
 +    opts.parseArgs(RestoreZookeeper.class.getName(), args);
 +    
 +    InputStream in = System.in;
 +    if (opts.file != null) {
 +      in = new FileInputStream(opts.file);
 +    }
 +    
 +    SAXParserFactory factory = SAXParserFactory.newInstance();
 +    SAXParser parser = factory.newSAXParser();
 +    parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
 +    in.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index 85f29ba,0000000..b1a553d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,277 -1,0 +1,278 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.File;
 +import java.io.FileInputStream;
 +import java.io.FileNotFoundException;
 +import java.io.FilenameFilter;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.io.UnsupportedEncodingException;
 +import java.net.Socket;
 +import java.net.URLEncoder;
 +import java.text.ParseException;
 +import java.text.SimpleDateFormat;
 +import java.util.Calendar;
 +import java.util.Date;
 +import java.util.Map;
 +import java.util.regex.Matcher;
 +import java.util.regex.Pattern;
 +
 +import javax.net.SocketFactory;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.commons.io.filefilter.WildcardFileFilter;
 +import org.apache.commons.lang.math.LongRange;
 +import org.apache.log4j.Category;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.spi.Filter;
 +import org.apache.log4j.spi.LocationInfo;
 +import org.apache.log4j.spi.LoggingEvent;
 +import org.apache.log4j.spi.ThrowableInformation;
 +import org.apache.log4j.varia.LevelRangeFilter;
 +import org.apache.log4j.xml.XMLLayout;
 +
 +import com.beust.jcommander.IStringConverter;
 +import com.beust.jcommander.Parameter;
 +
 +public class SendLogToChainsaw extends XMLLayout {
 +
 +  private static Pattern logPattern = Pattern.compile(
 +      "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
 +
 +  private File[] logFiles = null;
 +
 +  private SocketFactory factory = SocketFactory.getDefault();
 +
 +  private WildcardFileFilter fileFilter = null;
 +
 +  private Socket socket = null;
 +
 +  private Pattern lineFilter = null;
 +
 +  private LongRange dateFilter = null;
 +
 +  private LevelRangeFilter levelFilter = null;
 +
 +  public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
 +
 +    // Set up the file name filter
 +    if (null != fileNameFilter) {
 +      fileFilter = new WildcardFileFilter(fileNameFilter);
 +    } else {
 +      fileFilter = new WildcardFileFilter("*");
 +    }
 +
 +    // Get the list of files that match
 +    File dir = new File(directory);
 +    if (dir.isDirectory()) {
 +      logFiles = dir.listFiles((FilenameFilter) fileFilter);
 +    } else {
 +      throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
 +    }
 +
 +    if (logFiles.length == 0) {
 +      throw new IllegalArgumentException("No files match the supplied filter.");
 +    }
 +
 +    socket = factory.createSocket(host, port);
 +
 +    lineFilter = Pattern.compile(regex);
 +
 +    // Create Date Filter
 +    if (null != start) {
 +      if (end == null)
 +        end = new Date(System.currentTimeMillis());
 +      dateFilter = new LongRange(start.getTime(), end.getTime());
 +    }
 +
 +    if (null != level) {
 +      Level base = Level.toLevel(level.toUpperCase());
 +      levelFilter = new LevelRangeFilter();
 +      levelFilter.setAcceptOnMatch(true);
 +      levelFilter.setLevelMin(base);
 +      levelFilter.setLevelMax(Level.FATAL);
 +    }
 +  }
 +
 +  public void processLogFiles() throws IOException {
 +    String line = null;
 +    String out = null;
 +    InputStreamReader isReader = null;
 +    BufferedReader reader = null;
 +    try {
 +      for (File log : logFiles) {
 +        // Parse the server type and name from the log file name
 +        String threadName = log.getName().substring(0, log.getName().indexOf("."));
 +        try {
-           isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
++          isReader = new InputStreamReader(new FileInputStream(log), UTF_8);
 +        } catch (FileNotFoundException e) {
 +          System.out.println("Unable to find file: " + log.getAbsolutePath());
 +          throw e;
 +        }
 +        reader = new BufferedReader(isReader);
 +
 +        try {
 +          line = reader.readLine();
 +          while (null != line) {
 +            out = convertLine(line, threadName);
 +            if (null != out) {
 +              if (socket != null && socket.isConnected())
-                 socket.getOutputStream().write(out.getBytes(Constants.UTF8));
++                socket.getOutputStream().write(out.getBytes(UTF_8));
 +              else
 +                System.err.println("Unable to send data to transport");
 +            }
 +            line = reader.readLine();
 +          }
 +        } catch (IOException e) {
 +          System.out.println("Error processing line: " + line + ". Output was " + out);
 +          throw e;
 +        } finally {
 +          if (reader != null) {
 +            reader.close();
 +          }
 +          if (isReader != null) {
 +            isReader.close();
 +          }
 +        }
 +      }
 +    } finally {
 +      if (socket != null && socket.isConnected()) {
 +        socket.close();
 +      }
 +    }
 +  }
 +
 +  private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
 +    String result = null;
 +    Matcher m = logPattern.matcher(line);
 +    if (m.matches()) {
 +
 +      Calendar cal = Calendar.getInstance();
 +      cal.setTime(new Date(System.currentTimeMillis()));
 +      Integer date = Integer.parseInt(m.group(1));
 +      Integer hour = Integer.parseInt(m.group(2));
 +      Integer min = Integer.parseInt(m.group(3));
 +      Integer sec = Integer.parseInt(m.group(4));
 +      Integer ms = Integer.parseInt(m.group(5));
 +      String clazz = m.group(6);
 +      String level = m.group(7);
 +      String message = m.group(8);
 +      // Apply the regex filter if supplied
 +      if (null != lineFilter) {
 +        Matcher match = lineFilter.matcher(message);
 +        if (!match.matches())
 +          return null;
 +      }
 +      // URL encode the message
-       message = URLEncoder.encode(message, Constants.UTF8.name());
++      message = URLEncoder.encode(message, UTF_8.name());
 +      // Assume that we are processing logs from today.
 +      // If the date in the line is greater than today, then it must be
 +      // from the previous month.
 +      cal.set(Calendar.DATE, date);
 +      cal.set(Calendar.HOUR_OF_DAY, hour);
 +      cal.set(Calendar.MINUTE, min);
 +      cal.set(Calendar.SECOND, sec);
 +      cal.set(Calendar.MILLISECOND, ms);
 +      if (date > cal.get(Calendar.DAY_OF_MONTH)) {
 +        cal.add(Calendar.MONTH, -1);
 +      }
 +      long ts = cal.getTimeInMillis();
 +      // If this event is not between the start and end dates, then skip it.
 +      if (null != dateFilter && !dateFilter.containsLong(ts))
 +        return null;
 +      Category c = Logger.getLogger(clazz);
 +      Level l = Level.toLevel(level);
 +      LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
 +          (Map<?,?>) null);
 +      // Check the log level filter
 +      if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
 +        return null;
 +      }
 +      result = format(event);
 +    }
 +    return result;
 +  }
 +
 +  private static class DateConverter implements IStringConverter<Date> {
 +    @Override
 +    public Date convert(String value) {
 +      SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
 +      try {
 +        return formatter.parse(value);
 +      } catch (ParseException e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +
 +  }
 +
 +  private static class Opts extends Help {
 +
 +    @Parameter(names = {"-d", "--logDirectory"}, description = "ACCUMULO log directory path", required = true)
 +    String dir;
 +
 +    @Parameter(names = {"-f", "--fileFilter"}, description = "filter to apply to names of logs")
 +    String filter;
 +
 +    @Parameter(names = {"-h", "--host"}, description = "host where chainsaw is running", required = true)
 +    String hostname;
 +
 +    @Parameter(names = {"-p", "--port"}, description = "port where XMLSocketReceiver is listening", required = true)
 +    int portnum;
 +
 +    @Parameter(names = {"-s", "--start"}, description = "start date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
 +    Date startDate;
 +
 +    @Parameter(names = {"-e", "--end"}, description = "end date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
 +    Date endDate;
 +
 +    @Parameter(names = {"-l", "--level"}, description = "filter log level")
 +    String level;
 +
 +    @Parameter(names = {"-m", "--messageFilter"}, description = "regex filter for log messages")
 +    String regex;
 +  }
 +
 +  /**
 +   * 
 +   * @param args
 +   *          <ol>
 +   *          <li>path to log directory</li>
 +   *          <li>filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive)</li>
 +   *          <li>chainsaw host</li>
 +   *          <li>chainsaw port</li>
 +   *          <li>start date filter</li>
 +   *          <li>end date filter</li>
 +   *          <li>optional regex filter to match on each log4j message</li>
 +   *          <li>optional level filter</li>
 +   *          </ol>
 +   */
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    opts.parseArgs(SendLogToChainsaw.class.getName(), args);
 +
 +    SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
 +    c.processLogFiles();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index b6ca527,0000000..401d98c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@@ -1,45 -1,0 +1,47 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class SystemPropUtil {
 +  public static boolean setSystemProperty(String property, String value) throws KeeperException, InterruptedException {
 +    Property p = Property.getPropertyByKey(property);
 +    if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property))
 +      return false;
 +    
 +    // create the zk node for this property and set it's data to the specified value
 +    String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
-     ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    
 +    return true;
 +  }
 +  
 +  public static void removeSystemProperty(String property) throws InterruptedException, KeeperException {
 +    String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
 +    ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.FAIL);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index bcaf9b0,0000000..6ee1593
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class TablePropUtil {
 +  public static boolean setTableProperty(String tableId, String property, String value) throws KeeperException, InterruptedException {
 +    if (!isPropertyValid(property, value))
 +      return false;
 +    
 +    // create the zk node for per-table properties for this table if it doesn't already exist
 +    String zkTablePath = getTablePath(tableId);
 +    ZooReaderWriter.getInstance().putPersistentData(zkTablePath, new byte[0], NodeExistsPolicy.SKIP);
 +    
 +    // create the zk node for this property and set it's data to the specified value
 +    String zPath = zkTablePath + "/" + property;
-     ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    
 +    return true;
 +  }
 +  
 +  public static boolean isPropertyValid(String property, String value) {
 +    Property p = Property.getPropertyByKey(property);
 +    if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
 +      return false;
 +    
 +    return true;
 +  }
 +  
 +  public static void removeTableProperty(String tableId, String property) throws InterruptedException, KeeperException {
 +    String zPath = getTablePath(tableId) + "/" + property;
 +    ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
 +  }
 +  
 +  private static String getTablePath(String tablename) {
 +    return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tablename + Constants.ZTABLE_CONF;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 34c2151,0000000..edbb103
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,72 -1,0 +1,74 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.List;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class TabletServerLocks {
 +  
 +  static class Opts extends Help {
 +    @Parameter(names="-list")
 +    boolean list = false;
 +    @Parameter(names="-delete")
 +    String delete = null;
 +  }
 +  public static void main(String[] args) throws Exception {
 +    
 +    Instance instance = HdfsZooInstance.getInstance();
 +    String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +    Opts opts = new Opts();
 +    opts.parseArgs(TabletServerLocks.class.getName(), args);
 +    
 +    ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    
 +    if (opts.list) {
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      
 +      List<String> tabletServers = zoo.getChildren(tserverPath);
 +      
 +      for (String tabletServer : tabletServers) {
 +        byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
 +        String holder = null;
 +        if (lockData != null) {
-           holder = new String(lockData, Constants.UTF8);
++          holder = new String(lockData, UTF_8);
 +        }
 +        
 +        System.out.printf("%32s %16s%n", tabletServer, holder);
 +      }
 +    } else if (opts.delete != null) {
 +      ZooLock.deleteLock(tserverPath + "/" + args[1]);
 +    } else {
 +      System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
 +    }
 +    
 +  }
 +  
 +}


[32/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index 292129d,0000000..7ae4d5a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@@ -1,150 -1,0 +1,152 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.watcher;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Appender;
 +import org.apache.log4j.LogManager;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.helpers.FileWatchdog;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * Watcher that updates the monitor's log4j port from ZooKeeper in a system property
 + */
 +public class MonitorLog4jWatcher extends FileWatchdog implements Watcher {
 +  private static final Logger log = Logger.getLogger(MonitorLog4jWatcher.class);
 +
 +  private static final String HOST_PROPERTY_NAME = "org.apache.accumulo.core.host.log";
 +  private static final String PORT_PROPERTY_NAME = "org.apache.accumulo.core.host.log.port";
 +
 +  private final Object lock;
 +  private final Log4jConfiguration logConfig;
 +  private boolean loggingDisabled = false;
 +  protected String path;
 +
 +  public MonitorLog4jWatcher(String instance, String filename) {
 +    super(filename);
 +    this.path = ZooUtil.getRoot(instance) + Constants.ZMONITOR_LOG4J_ADDR;
 +    this.lock = new Object();
 +    this.logConfig = new Log4jConfiguration(filename);
 +    doOnChange();
 +  }
 +
 +  boolean isUsingProperties() {
 +    return logConfig.isUsingProperties();
 +  }
 +
 +  String getPath() {
 +    return path;
 +  }
 +
 +  @Override
 +  public void run() {
 +    try {
 +      // Initially set the logger if the Monitor's log4j advertisement node exists
 +      if (ZooReaderWriter.getInstance().exists(path, this))
 +        updateMonitorLog4jLocation();
 +      log.info("Set watch for Monitor Log4j watcher");
 +    } catch (Exception e) {
 +      log.error("Unable to set watch for Monitor Log4j watcher on " + path);
 +    }
 +
 +    super.run();
 +  }
 +
 +  @Override
 +  public void doOnChange() {
 +    // this method gets called in the parent class' constructor
 +    // I'm not sure of a better way to get around this. The final modifier helps though.
 +    if (null == lock) {
 +      return;
 +    }
 +
 +    synchronized (lock) {
 +      // We might triggered by file-reloading or from ZK update.
 +      // Either way will result in log-forwarding being restarted
 +      loggingDisabled = false;
 +      log.info("Enabled log-forwarding");
 +      logConfig.resetLogger();
 +    }
 +  }
 +
 +  @Override
 +  public void process(WatchedEvent event) {
 +    // We got an update, process the data in the node
 +    updateMonitorLog4jLocation();
 +
 +    if (event.getPath() != null) {
 +      try {
 +        ZooReaderWriter.getInstance().exists(event.getPath(), this);
 +      } catch (Exception ex) {
 +        log.error("Unable to reset watch for Monitor Log4j watcher", ex);
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Read the host and port information for the Monitor's log4j socket and update the system properties so that, on logger refresh, it sees the new information.
 +   */
 +  protected void updateMonitorLog4jLocation() {
 +    try {
-       String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), Constants.UTF8);
++      String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), UTF_8);
 +      HostAndPort hostAndPort = HostAndPort.fromString(hostPortString);
 +
 +      System.setProperty(HOST_PROPERTY_NAME, hostAndPort.getHostText());
 +      System.setProperty(PORT_PROPERTY_NAME, Integer.toString(hostAndPort.getPort()));
 +
 +      log.info("Changing monitor log4j address to " + hostAndPort.toString());
 +
 +      doOnChange();
 +    } catch (NoNodeException e) {
 +      // Not sure on the synchronization guarantees for Loggers and Appenders
 +      // on configuration reload
 +      synchronized (lock) {
 +        // Don't need to try to re-disable'ing it.
 +        if (loggingDisabled) {
 +          return;
 +        }
 +
 +        Logger logger = LogManager.getLogger("org.apache.accumulo");
 +        if (null != logger) {
 +          // TODO ACCUMULO-2343 Create a specific appender for log-forwarding to the monitor
 +          // that can replace the AsyncAppender+SocketAppender.
 +          Appender appender = logger.getAppender("ASYNC");
 +          if (null != appender) {
 +            log.info("Closing log-forwarding appender");
 +            appender.close();
 +            log.info("Removing log-forwarding appender");
 +            logger.removeAppender(appender);
 +            loggingDisabled = true;
 +          }
 +        }
 +      }
 +    } catch (IllegalArgumentException e) {
 +      log.error("Could not parse host and port information", e);
 +    } catch (Exception e) {
 +      log.error("Error reading zookeeper data for Monitor Log4j watcher", e);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
index b02e5d4,0000000..1bbae03
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
@@@ -1,54 -1,0 +1,55 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.zookeeper;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.locks.Lock;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class ZooQueueLock extends org.apache.accumulo.fate.zookeeper.ZooQueueLock {
 +
 +  public ZooQueueLock(String path, boolean ephemeral) throws KeeperException, InterruptedException {
 +    super(ZooReaderWriter.getInstance(), path, ephemeral);
 +  }
 +
 +  public static void main(String args[]) throws InterruptedException, KeeperException {
 +    ZooQueueLock lock = new ZooQueueLock("/lock", true);
-     DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(Constants.UTF8));
-     DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(Constants.UTF8));
++    DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(UTF_8));
++    DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(UTF_8));
 +    final Lock readLock = rlocker.readLock();
 +    readLock.lock();
 +    final Lock readLock2 = rlocker.readLock();
 +    readLock2.lock();
 +    final Lock writeLock = wlocker.writeLock();
 +    if (writeLock.tryLock(100, TimeUnit.MILLISECONDS))
 +      throw new RuntimeException("Write lock achieved during read lock!");
 +    readLock.unlock();
 +    readLock2.unlock();
 +    writeLock.lock();
 +    if (readLock.tryLock(100, TimeUnit.MILLISECONDS))
 +      throw new RuntimeException("Read lock achieved during write lock!");
-     final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(Constants.UTF8));
++    final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(UTF_8));
 +    writeLock2.unlock();
 +    readLock.lock();
 +    System.out.println("success");
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
index 435591d,0000000..c5374a2
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
@@@ -1,42 -1,0 +1,43 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.zookeeper;
 +
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +
 +public class ZooReaderWriter extends org.apache.accumulo.fate.zookeeper.ZooReaderWriter {
 +  private static final String SCHEME = "digest";
 +  private static final String USER = "accumulo";
 +  private static ZooReaderWriter instance = null;
 +  
 +  public ZooReaderWriter(String string, int timeInMillis, String secret) {
-     super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++    super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
 +  }
 +  
 +  public static synchronized ZooReaderWriter getInstance() {
 +    if (instance == null) {
 +      AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
 +      instance = new ZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
 +          conf.get(Property.INSTANCE_SECRET));
 +    }
 +    return instance;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index 4a7111d,0000000..e15e63e
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@@ -1,64 -1,0 +1,65 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.zookeeper;
 +
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +
 +/**
 + * A factory for {@link ZooReaderWriter} objects.
 + */
 +public class ZooReaderWriterFactory {
 +  private static final String SCHEME = "digest";
 +  private static final String USER = "accumulo";
 +  private static IZooReaderWriter instance = null;
 +
 +  /**
 +   * Gets a new reader/writer.
 +   *
 +   * @param string
 +   *          ZooKeeper connection string
 +   * @param timeInMillis
 +   *          session timeout in milliseconds
 +   * @param secret
 +   *          instance secret
 +   * @return reader/writer
 +   */
 +  public IZooReaderWriter getZooReaderWriter(String string, int timeInMillis, String secret) {
-     return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++    return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
 +  }
 +
 +  /**
 +   * Gets a reader/writer, retrieving ZooKeeper information from the site configuration. The same instance may be returned for multiple calls.
 +   *
 +   * @return reader/writer
 +   */
 +  public IZooReaderWriter getInstance() {
 +    synchronized (ZooReaderWriterFactory.class) {
 +      if (instance == null) {
 +        AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
 +        instance = getZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
 +            conf.get(Property.INSTANCE_SECRET));
 +      }
 +      return instance;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index e7f42af,0000000..3aed7ef
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@@ -1,159 -1,0 +1,161 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collection;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.UUID;
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.Before;
 +import org.junit.Test;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.eq;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.easymock.EasyMock.verify;
 +
 +public class NamespaceConfigurationTest {
 +  private static final String NSID = "namespace";
 +  private static final String ZOOKEEPERS = "localhost";
 +  private static final int ZK_SESSION_TIMEOUT = 120000;
 +
 +  private String iid;
 +  private Instance instance;
 +  private AccumuloConfiguration parent;
 +  private ZooCacheFactory zcf;
 +  private ZooCache zc;
 +  private NamespaceConfiguration c;
 +
 +  @Before
 +  public void setUp() {
 +    iid = UUID.randomUUID().toString();
 +    instance = createMock(Instance.class);
 +    parent = createMock(AccumuloConfiguration.class);
 +    c = new NamespaceConfiguration(NSID, instance, parent);
 +    zcf = createMock(ZooCacheFactory.class);
 +    c.setZooCacheFactory(zcf);
 +
 +    expect(instance.getInstanceID()).andReturn(iid);
 +    expectLastCall().anyTimes();
 +    expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
 +    expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
 +    replay(instance);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
 +    replay(zcf);
 +  }
 +
 +  @Test
 +  public void testGetters() {
 +    assertEquals(NSID, c.getNamespaceId());
 +    assertEquals(parent, c.getParentConfiguration());
 +  }
 +
 +  @Test
 +  public void testGet_InZK() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
-         "sekrit".getBytes(Constants.UTF8));
++        "sekrit".getBytes(UTF_8));
 +    replay(zc);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGet_InParent() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(null);
 +    replay(zc);
 +    expect(parent.get(p)).andReturn("sekrit");
 +    replay(parent);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGet_SkipParentIfAccumuloNS() {
 +    c = new NamespaceConfiguration(Namespaces.ACCUMULO_NAMESPACE_ID, instance, parent);
 +    c.setZooCacheFactory(zcf);
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + Namespaces.ACCUMULO_NAMESPACE_ID + Constants.ZNAMESPACE_CONF + "/" + p.getKey()))
 +        .andReturn(null);
 +    replay(zc);
 +    assertNull(c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGetProperties() {
 +    PropertyFilter filter = new AllFilter();
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add("foo");
 +    children.add("ding");
 +    expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF)).andReturn(children);
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "foo")).andReturn(
-         "bar".getBytes(Constants.UTF8));
++        "bar".getBytes(UTF_8));
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "ding")).andReturn(
-         "dong".getBytes(Constants.UTF8));
++        "dong".getBytes(UTF_8));
 +    replay(zc);
 +    c.getProperties(props, filter);
 +    assertEquals(2, props.size());
 +    assertEquals("bar", props.get("foo"));
 +    assertEquals("dong", props.get("ding"));
 +  }
 +
 +  @Test
 +  public void testObserver() {
 +    ConfigurationObserver o = createMock(ConfigurationObserver.class);
 +    c.addObserver(o);
 +    Collection<ConfigurationObserver> os = c.getObservers();
 +    assertEquals(1, os.size());
 +    assertTrue(os.contains(o));
 +    c.removeObserver(o);
 +    os = c.getObservers();
 +    assertEquals(0, os.size());
 +  }
 +
 +  @Test
 +  public void testInvalidateCache() {
 +    // need to do a get so the accessor is created
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
-         "sekrit".getBytes(Constants.UTF8));
++        "sekrit".getBytes(UTF_8));
 +    zc.clear();
 +    replay(zc);
 +    c.get(Property.INSTANCE_SECRET);
 +    c.invalidateCache();
 +    verify(zc);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
index 7f07d94,0000000..81148a0
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
@@@ -1,134 -1,0 +1,134 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.endsWith;
 +import static org.easymock.EasyMock.eq;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertSame;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.After;
 +import org.junit.Before;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +public class ServerConfigurationFactoryTest {
 +  private static final String ZK_HOST = "localhost";
 +  private static final int ZK_TIMEOUT = 120000;
 +  private static final String IID = "iid";
 +
 +  // use the same mock ZooCacheFactory and ZooCache for all tests
 +  private static ZooCacheFactory zcf;
 +  private static ZooCache zc;
 +
 +  @BeforeClass
 +  public static void setUpClass() throws Exception {
 +    zcf = createMock(ZooCacheFactory.class);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache(eq(ZK_HOST), eq(ZK_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
 +    expectLastCall().anyTimes();
 +    expect(zcf.getZooCache(ZK_HOST, ZK_TIMEOUT)).andReturn(zc);
 +    expectLastCall().anyTimes();
 +    replay(zcf);
 +
 +    expect(zc.getChildren(anyObject(String.class))).andReturn(null);
 +    expectLastCall().anyTimes();
 +    // ConfigSanityCheck looks at timeout
-     expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(Constants.UTF8));
++    expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(UTF_8));
 +    replay(zc);
 +  }
 +
 +  private Instance instance;
 +  private ServerConfigurationFactory scf;
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    instance = createMock(Instance.class);
 +    expect(instance.getInstanceID()).andReturn(IID);
 +    expectLastCall().anyTimes();
 +  }
 +
 +  @After
 +  public void tearDown() throws Exception {
 +    ServerConfigurationFactory.clearCachedConfigurations();
 +  }
 +
 +  private void mockInstanceForConfig() {
 +    expect(instance.getZooKeepers()).andReturn(ZK_HOST);
 +    expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_TIMEOUT);
 +  }
 +
 +  private void ready() {
 +    replay(instance);
 +    scf = new ServerConfigurationFactory(instance);
 +    scf.setZooCacheFactory(zcf);
 +  }
 +
 +  @Test
 +  public void testGetInstance() {
 +    ready();
 +    assertSame(instance, scf.getInstance());
 +  }
 +
 +  @Test
 +  public void testGetDefaultConfiguration() {
 +    ready();
 +    DefaultConfiguration c = scf.getDefaultConfiguration();
 +    assertNotNull(c);
 +  }
 +
 +  @Test
 +  public void testGetSiteConfiguration() {
 +    ready();
 +    SiteConfiguration c = scf.getSiteConfiguration();
 +    assertNotNull(c);
 +  }
 +
 +  @Test
 +  public void testGetConfiguration() {
 +    mockInstanceForConfig();
 +    ready();
 +    AccumuloConfiguration c = scf.getConfiguration();
 +    assertNotNull(c);
 +  }
 +
 +  private static final String NSID = "NAMESPACE";
 +
 +  @Test
 +  public void testGetNamespaceConfiguration() {
 +    mockInstanceForConfig();
 +    ready();
 +    NamespaceConfiguration c = scf.getNamespaceConfiguration(NSID);
 +    assertEquals(NSID, c.getNamespaceId());
 +
 +    assertSame(c, scf.getNamespaceConfiguration(NSID));
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
index 92580f3,0000000..2f38e4c
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
@@@ -1,145 -1,0 +1,146 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.eq;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.easymock.EasyMock.verify;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.Collection;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class TableConfigurationTest {
 +  private static final String TID = "table";
 +  private static final String ZOOKEEPERS = "localhost";
 +  private static final int ZK_SESSION_TIMEOUT = 120000;
 +
 +  private String iid;
 +  private Instance instance;
 +  private NamespaceConfiguration parent;
 +  private ZooCacheFactory zcf;
 +  private ZooCache zc;
 +  private TableConfiguration c;
 +
 +  @Before
 +  public void setUp() {
 +    iid = UUID.randomUUID().toString();
 +    instance = createMock(Instance.class);
 +    parent = createMock(NamespaceConfiguration.class);
 +    c = new TableConfiguration(iid, instance, TID, parent);
 +    zcf = createMock(ZooCacheFactory.class);
 +    c.setZooCacheFactory(zcf);
 +
 +    expect(instance.getInstanceID()).andReturn(iid);
 +    expectLastCall().anyTimes();
 +    expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
 +    expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
 +    replay(instance);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(TableConfWatcher.class))).andReturn(zc);
 +    replay(zcf);
 +  }
 +
 +  @Test
 +  public void testGetters() {
 +    assertEquals(TID, c.getTableId());
 +    assertEquals(parent, c.getParentConfiguration());
 +  }
 +
 +  @Test
 +  public void testGet_InZK() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
-         .andReturn("sekrit".getBytes(Constants.UTF8));
++        .andReturn("sekrit".getBytes(UTF_8));
 +    replay(zc);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGet_InParent() {
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(null);
 +    replay(zc);
 +    expect(parent.get(p)).andReturn("sekrit");
 +    replay(parent);
 +    assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
 +  }
 +
 +  @Test
 +  public void testGetProperties() {
 +    PropertyFilter filter = new AllFilter();
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add("foo");
 +    children.add("ding");
 +    expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF)).andReturn(children);
-     expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(Constants.UTF8));
-     expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(Constants.UTF8));
++    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(UTF_8));
++    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(UTF_8));
 +    replay(zc);
 +    c.getProperties(props, filter);
 +    assertEquals(2, props.size());
 +    assertEquals("bar", props.get("foo"));
 +    assertEquals("dong", props.get("ding"));
 +  }
 +
 +  @Test
 +  public void testObserver() {
 +    ConfigurationObserver o = createMock(ConfigurationObserver.class);
 +    c.addObserver(o);
 +    Collection<ConfigurationObserver> os = c.getObservers();
 +    assertEquals(1, os.size());
 +    assertTrue(os.contains(o));
 +    c.removeObserver(o);
 +    os = c.getObservers();
 +    assertEquals(0, os.size());
 +  }
 +
 +  @Test
 +  public void testInvalidateCache() {
 +    // need to do a get so the accessor is created
 +    Property p = Property.INSTANCE_SECRET;
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
-         .andReturn("sekrit".getBytes(Constants.UTF8));
++        .andReturn("sekrit".getBytes(UTF_8));
 +    zc.clear();
 +    replay(zc);
 +    c.get(Property.INSTANCE_SECRET);
 +    c.invalidateCache();
 +    verify(zc);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
index 1b9d8f7,0000000..4369119
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
@@@ -1,186 -1,0 +1,186 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.replay;
 +import static org.easymock.EasyMock.verify;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertSame;
 +
 +import java.util.List;
 +import java.util.Map;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ZooCachePropertyAccessorTest {
 +  private static final String PATH = "/root/path/to/props";
 +  private static final Property PROP = Property.INSTANCE_SECRET;
 +  private static final String KEY = PROP.getKey();
 +  private static final String FULL_PATH = PATH + "/" + KEY;
 +  private static final String VALUE = "value";
-   private static final byte[] VALUE_BYTES = VALUE.getBytes(Constants.UTF8);
++  private static final byte[] VALUE_BYTES = VALUE.getBytes(UTF_8);
 +
 +  private ZooCache zc;
 +  private ZooCachePropertyAccessor a;
 +
 +  @Before
 +  public void setUp() {
 +    zc = createMock(ZooCache.class);
 +    a = new ZooCachePropertyAccessor(zc);
 +  }
 +
 +  @Test
 +  public void testGetter() {
 +    assertSame(zc, a.getZooCache());
 +  }
 +
 +  @Test
 +  public void testGet_Valid() {
 +    expect(zc.get(FULL_PATH)).andReturn(VALUE_BYTES);
 +    replay(zc);
 +    assertEquals(VALUE, a.get(PROP, PATH, null));
 +  }
 +
 +  @Test
 +  public void testGet_Parent() {
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    expect(parent.get(PROP)).andReturn(VALUE);
 +    replay(parent);
 +    expect(zc.get(FULL_PATH)).andReturn(null);
 +    replay(zc);
 +    assertEquals(VALUE, a.get(PROP, PATH, parent));
 +  }
 +
 +  @Test
 +  public void testGet_Parent_Null() {
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    expect(parent.get(PROP)).andReturn(null);
 +    replay(parent);
 +    expect(zc.get(FULL_PATH)).andReturn(null);
 +    replay(zc);
 +    assertNull(a.get(PROP, PATH, parent));
 +  }
 +
 +  @Test
 +  public void testGet_Null_NoParent() {
 +    expect(zc.get(FULL_PATH)).andReturn(null);
 +    replay(zc);
 +    assertNull(a.get(PROP, PATH, null));
 +  }
 +
 +  @Test
 +  public void testGet_InvalidFormat() {
 +    Property badProp = Property.MASTER_CLIENTPORT;
 +    expect(zc.get(PATH + "/" + badProp.getKey())).andReturn(VALUE_BYTES);
 +    replay(zc);
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    expect(parent.get(badProp)).andReturn("12345");
 +    replay(parent);
 +    assertEquals("12345", a.get(badProp, PATH, parent));
 +  }
 +
 +  @Test
 +  public void testGetProperties() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    String child1 = "child1";
 +    String child2 = "child2";
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add(child1);
 +    children.add(child2);
 +    expect(zc.getChildren(PATH)).andReturn(children);
 +    expect(zc.get(PATH + "/" + child1)).andReturn(VALUE_BYTES);
 +    expect(zc.get(PATH + "/" + child2)).andReturn(null);
 +    replay(zc);
 +    expect(filter.accept(child1)).andReturn(true);
 +    expect(filter.accept(child2)).andReturn(true);
 +    replay(filter);
 +
 +    a.getProperties(props, PATH, filter, parent, null);
 +    assertEquals(1, props.size());
 +    assertEquals(VALUE, props.get(child1));
 +    verify(parent);
 +  }
 +
 +  @Test
 +  public void testGetProperties_NoChildren() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    expect(zc.getChildren(PATH)).andReturn(null);
 +    replay(zc);
 +
 +    a.getProperties(props, PATH, filter, parent, null);
 +    assertEquals(0, props.size());
 +  }
 +
 +  @Test
 +  public void testGetProperties_Filter() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, filter);
 +    replay(parent);
 +    String child1 = "child1";
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add(child1);
 +    expect(zc.getChildren(PATH)).andReturn(children);
 +    replay(zc);
 +    expect(filter.accept(child1)).andReturn(false);
 +    replay(filter);
 +
 +    a.getProperties(props, PATH, filter, parent, null);
 +    assertEquals(0, props.size());
 +  }
 +
 +  @Test
 +  public void testGetProperties_ParentFilter() {
 +    Map<String,String> props = new java.util.HashMap<String,String>();
 +    AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
 +    PropertyFilter filter = createMock(PropertyFilter.class);
 +    PropertyFilter parentFilter = createMock(PropertyFilter.class);
 +    parent.getProperties(props, parentFilter);
 +    replay(parent);
 +    expect(zc.getChildren(PATH)).andReturn(null);
 +    replay(zc);
 +
 +    a.getProperties(props, PATH, filter, parent, parentFilter);
 +    verify(parent);
 +  }
 +
 +  @Test
 +  public void testInvalidateCache() {
 +    zc.clear();
 +    replay(zc);
 +    a.invalidateCache();
 +    verify(zc);
 +  }
 +}


[42/50] [abbrv] git commit: Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
Merge branch '1.5' into 1.6


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9b20a9d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9b20a9d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9b20a9d4

Branch: refs/heads/1.6
Commit: 9b20a9d4062f9ac0d34ce4b2acd7b9b72524a84f
Parents: 5e0f6a4 2deabd3
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Oct 31 23:37:27 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Oct 31 23:37:27 2014 -0400

----------------------------------------------------------------------
 assemble/bin/bootstrap_config.sh                |  2 +-
 .../org/apache/accumulo/core/Constants.java     | 13 +++++-
 .../apache/accumulo/core/cli/ClientOpts.java    |  6 ++-
 .../accumulo/core/client/BatchWriterConfig.java | 11 ++---
 .../accumulo/core/client/ZooKeeperInstance.java | 14 ++++---
 .../client/impl/InstanceOperationsImpl.java     |  4 +-
 .../accumulo/core/client/impl/Namespaces.java   |  6 ++-
 .../accumulo/core/client/impl/ServerClient.java |  4 +-
 .../core/client/impl/TableOperationsImpl.java   | 32 ++++++++-------
 .../accumulo/core/client/impl/Tables.java       | 10 +++--
 .../accumulo/core/client/impl/Writer.java       |  5 ++-
 .../core/client/lexicoder/StringLexicoder.java  |  6 +--
 .../core/client/mapreduce/RangeInputSplit.java  |  7 ++--
 .../mapreduce/lib/impl/ConfiguratorBase.java    |  4 +-
 .../mapreduce/lib/impl/InputConfigurator.java   |  8 ++--
 .../mapreduce/lib/impl/OutputConfigurator.java  |  7 ++--
 .../lib/partition/RangePartitioner.java         |  7 ++--
 .../accumulo/core/client/mock/MockShell.java    | 11 ++---
 .../client/security/tokens/PasswordToken.java   |  5 ++-
 .../accumulo/core/conf/ConfigurationDocGen.java |  9 ++--
 .../accumulo/core/data/ArrayByteSequence.java   |  8 ++--
 .../org/apache/accumulo/core/data/Column.java   |  6 +--
 .../apache/accumulo/core/data/Condition.java    |  9 ++--
 .../org/apache/accumulo/core/data/Value.java    |  4 +-
 .../accumulo/core/file/BloomFilterLayer.java    |  7 ++--
 .../core/iterators/ColumnFamilyCounter.java     |  5 ++-
 .../accumulo/core/iterators/LongCombiner.java   |  7 ++--
 .../accumulo/core/iterators/conf/ColumnSet.java |  7 ++--
 .../core/iterators/user/BigDecimalCombiner.java |  7 ++--
 .../core/iterators/user/GrepIterator.java       |  5 ++-
 .../iterators/user/IntersectingIterator.java    |  7 ++--
 .../core/iterators/user/LargeRowFilter.java     |  5 ++-
 .../core/iterators/user/RegExFilter.java        |  7 ++--
 .../iterators/user/RowDeletingIterator.java     |  5 ++-
 .../iterators/user/SummingArrayCombiner.java    |  7 ++--
 .../iterators/user/TransformingIterator.java    |  7 ++--
 .../core/iterators/user/VisibilityFilter.java   |  5 ++-
 .../accumulo/core/security/Authorizations.java  | 13 +++---
 .../core/security/ColumnVisibility.java         | 43 ++++++++++----------
 .../accumulo/core/security/Credentials.java     | 11 ++---
 .../core/security/VisibilityConstraint.java     |  5 ++-
 .../core/security/VisibilityParseException.java |  6 +--
 .../core/trace/InstanceUserPassword.java        |  5 ++-
 .../accumulo/core/trace/ZooTraceClient.java     |  5 ++-
 .../core/util/ByteArrayBackedCharSequence.java  |  5 ++-
 .../apache/accumulo/core/util/ByteArraySet.java |  6 +--
 .../accumulo/core/util/ByteBufferUtil.java      |  5 ++-
 .../apache/accumulo/core/util/CreateToken.java  |  5 ++-
 .../org/apache/accumulo/core/util/Encoding.java |  5 ++-
 .../apache/accumulo/core/util/FastFormat.java   |  4 +-
 .../org/apache/accumulo/core/util/Merge.java    |  5 ++-
 .../apache/accumulo/core/util/MonitorUtil.java  |  4 +-
 .../org/apache/accumulo/core/util/TextUtil.java |  4 +-
 .../apache/accumulo/core/util/shell/Shell.java  |  6 ++-
 .../util/shell/commands/AddSplitsCommand.java   |  7 ++--
 .../shell/commands/AuthenticateCommand.java     |  5 ++-
 .../util/shell/commands/CreateTableCommand.java |  7 ++--
 .../util/shell/commands/ExecfileCommand.java    |  5 ++-
 .../core/util/shell/commands/HiddenCommand.java |  5 ++-
 .../core/util/shell/commands/PasswdCommand.java |  5 ++-
 .../shell/commands/QuotedStringTokenizer.java   |  5 ++-
 .../core/util/shell/commands/UserCommand.java   |  5 ++-
 .../core/client/BatchWriterConfigTest.java      |  6 +--
 .../core/client/ZooKeeperInstanceTest.java      |  9 ++--
 .../core/client/mock/MockNamespacesTest.java    |  4 +-
 .../tokens/CredentialProviderTokenTest.java     |  7 ++--
 .../security/tokens/PasswordTokenTest.java      |  7 ++--
 .../accumulo/core/data/ConditionTest.java       | 12 +++---
 .../core/data/ConditionalMutationTest.java      | 10 ++---
 .../apache/accumulo/core/data/ValueTest.java    |  5 ++-
 .../core/iterators/user/RegExFilterTest.java    |  7 ++--
 .../core/security/ColumnVisibilityTest.java     |  6 +--
 .../accumulo/core/security/CredentialsTest.java |  6 +--
 .../core/security/VisibilityConstraintTest.java |  4 +-
 .../security/crypto/BlockedIOStreamTest.java    | 10 ++---
 .../core/util/format/DeleterFormatterTest.java  | 12 +++---
 .../ShardedTableDistributionFormatterTest.java  |  8 ++--
 .../org/apache/accumulo/fate/AdminUtil.java     |  8 ++--
 .../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++----
 .../zookeeper/DistributedReadWriteLock.java     | 18 ++++----
 .../accumulo/fate/zookeeper/ZooCache.java       |  8 ++--
 .../apache/accumulo/fate/zookeeper/ZooLock.java |  6 +--
 .../accumulo/fate/zookeeper/ZooReservation.java |  9 ++--
 .../accumulo/fate/zookeeper/ZooSession.java     |  6 +--
 .../impl/MiniAccumuloClusterImpl.java           |  4 +-
 .../org/apache/accumulo/proxy/ProxyServer.java  |  7 ++--
 .../apache/accumulo/proxy/TestProxyClient.java  | 15 +++----
 .../java/org/apache/accumulo/proxy/Util.java    |  5 ++-
 .../org/apache/accumulo/server/Accumulo.java    |  4 +-
 .../accumulo/server/client/HdfsZooInstance.java | 10 +++--
 .../server/conf/ZooCachePropertyAccessor.java   |  5 ++-
 .../accumulo/server/conf/ZooConfiguration.java  |  4 +-
 .../server/constraints/MetadataConstraints.java | 14 ++++---
 .../apache/accumulo/server/init/Initialize.java | 15 +++----
 .../accumulo/server/master/LiveTServerSet.java  |  3 +-
 .../server/master/state/DeadServerList.java     |  7 ++--
 .../server/master/state/TServerInstance.java    |  7 ++--
 .../master/state/TabletStateChangeIterator.java |  5 ++-
 .../accumulo/server/master/state/ZooStore.java  |  5 ++-
 .../master/state/ZooTabletStateStore.java       | 11 ++---
 .../server/metrics/AbstractMetricsImpl.java     |  5 ++-
 .../accumulo/server/monitor/LogService.java     |  4 +-
 .../server/security/SecurityOperation.java      |  4 +-
 .../server/security/SystemCredentials.java      | 10 +++--
 .../security/handler/ZKAuthenticator.java       |  4 +-
 .../server/security/handler/ZKAuthorizor.java   |  5 ++-
 .../server/security/handler/ZKPermHandler.java  |  5 ++-
 .../accumulo/server/tables/TableManager.java    | 18 ++++----
 .../server/tablets/UniqueNameAllocator.java     | 10 +++--
 .../server/util/AddFilesWithMissingEntries.java |  5 ++-
 .../accumulo/server/util/ChangeSecret.java      |  5 ++-
 .../accumulo/server/util/CleanZookeeper.java    |  6 ++-
 .../accumulo/server/util/DeleteZooInstance.java |  6 ++-
 .../accumulo/server/util/DumpZookeeper.java     |  5 ++-
 .../accumulo/server/util/ListInstances.java     |  6 ++-
 .../accumulo/server/util/MetadataTableUtil.java | 32 ++++++++-------
 .../accumulo/server/util/NamespacePropUtil.java |  4 +-
 .../accumulo/server/util/RandomizeVolumes.java  |  6 +--
 .../accumulo/server/util/RestoreZookeeper.java  |  7 ++--
 .../accumulo/server/util/SendLogToChainsaw.java |  9 ++--
 .../accumulo/server/util/SystemPropUtil.java    |  4 +-
 .../accumulo/server/util/TablePropUtil.java     |  4 +-
 .../accumulo/server/util/TabletServerLocks.java |  4 +-
 .../server/watcher/MonitorLog4jWatcher.java     |  4 +-
 .../accumulo/server/zookeeper/ZooQueueLock.java |  9 ++--
 .../server/zookeeper/ZooReaderWriter.java       |  5 ++-
 .../zookeeper/ZooReaderWriterFactory.java       |  5 ++-
 .../server/conf/NamespaceConfigurationTest.java | 10 +++--
 .../conf/ServerConfigurationFactoryTest.java    |  4 +-
 .../server/conf/TableConfigurationTest.java     |  9 ++--
 .../conf/ZooCachePropertyAccessorTest.java      |  4 +-
 .../java/org/apache/accumulo/master/Master.java |  8 ++--
 .../master/recovery/RecoveryManager.java        |  4 +-
 .../accumulo/master/state/SetGoalState.java     |  4 +-
 .../accumulo/master/tableOps/BulkImport.java    |  8 ++--
 .../master/tableOps/CancelCompactions.java      | 10 +++--
 .../accumulo/master/tableOps/CompactRange.java  | 14 ++++---
 .../accumulo/master/tableOps/ExportTable.java   |  8 ++--
 .../accumulo/master/tableOps/ImportTable.java   | 16 ++++----
 .../accumulo/master/tableOps/RenameTable.java   |  6 ++-
 .../apache/accumulo/master/tableOps/Utils.java  | 14 ++++---
 .../master/tserverOps/ShutdownTServer.java      |  4 +-
 .../org/apache/accumulo/monitor/Monitor.java    |  6 ++-
 .../accumulo/monitor/ZooKeeperStatus.java       |  7 ++--
 .../accumulo/monitor/servlets/BasicServlet.java | 10 +++--
 .../monitor/servlets/DefaultServlet.java        |  5 ++-
 .../accumulo/monitor/servlets/ShellServlet.java |  5 ++-
 .../accumulo/monitor/servlets/XMLServlet.java   |  7 ++--
 .../accumulo/monitor/servlets/trace/Basic.java  |  5 ++-
 .../org/apache/accumulo/tracer/TraceServer.java |  8 ++--
 .../tserver/BulkFailedCopyProcessor.java        |  5 ++-
 .../org/apache/accumulo/tserver/Tablet.java     | 10 +++--
 .../apache/accumulo/tserver/TabletServer.java   |  9 ++--
 .../apache/accumulo/tserver/log/DfsLogger.java  |  4 +-
 .../accumulo/tserver/logger/LogFileValue.java   | 11 ++---
 .../accumulo/tserver/logger/LogReader.java      |  5 ++-
 .../accumulo/test/BulkImportDirectory.java      |  5 ++-
 .../accumulo/test/NativeMapConcurrencyTest.java |  5 ++-
 .../accumulo/test/NativeMapPerformanceTest.java |  7 ++--
 .../accumulo/test/NativeMapStressTest.java      |  5 ++-
 .../apache/accumulo/test/TestBinaryRows.java    |  7 ++--
 .../org/apache/accumulo/test/TestIngest.java    |  7 ++--
 .../accumulo/test/TestMultiTableIngest.java     |  5 ++-
 .../test/continuous/ContinuousIngest.java       |  8 ++--
 .../test/continuous/ContinuousMoru.java         |  7 ++--
 .../test/continuous/ContinuousQuery.java        |  5 ++-
 .../test/continuous/ContinuousScanner.java      |  5 ++-
 .../test/continuous/ContinuousVerify.java       |  7 ++--
 .../test/continuous/ContinuousWalk.java         | 17 ++++----
 .../accumulo/test/continuous/Histogram.java     |  6 +--
 .../test/continuous/PrintScanTimeHistogram.java |  5 ++-
 .../accumulo/test/continuous/TimeBinner.java    |  5 ++-
 .../test/continuous/UndefinedAnalyzer.java      |  9 ++--
 .../test/functional/CacheTestReader.java        |  7 ++--
 .../test/functional/CacheTestWriter.java        | 11 ++---
 .../accumulo/test/functional/ZombieTServer.java |  4 +-
 .../metadata/MetadataBatchScanTest.java         |  7 ++--
 .../apache/accumulo/test/randomwalk/Module.java |  7 ++--
 .../test/randomwalk/bulk/BulkMinusOne.java      |  5 ++-
 .../accumulo/test/randomwalk/bulk/Verify.java   |  5 ++-
 .../test/randomwalk/concurrent/BatchWrite.java  |  5 ++-
 .../test/randomwalk/concurrent/BulkImport.java  |  5 ++-
 .../concurrent/ChangeAuthorizations.java        |  7 ++--
 .../randomwalk/concurrent/StopTabletServer.java |  4 +-
 .../accumulo/test/randomwalk/image/Verify.java  |  5 ++-
 .../accumulo/test/randomwalk/image/Write.java   |  9 ++--
 .../test/randomwalk/multitable/Write.java       |  5 ++-
 .../randomwalk/security/SecurityHelper.java     |  7 ++--
 .../test/randomwalk/security/TableOp.java       |  9 ++--
 .../test/randomwalk/sequential/Write.java       |  5 ++-
 .../test/randomwalk/shard/BulkInsert.java       |  5 ++-
 .../test/randomwalk/shard/ExportIndex.java      |  5 ++-
 .../accumulo/test/scalability/Ingest.java       |  5 ++-
 .../apache/accumulo/test/AuditMessageIT.java    |  6 +--
 .../org/apache/accumulo/test/NamespacesIT.java  |  8 ++--
 .../java/org/apache/accumulo/test/VolumeIT.java |  6 +--
 .../accumulo/test/functional/AddSplitIT.java    |  5 ++-
 .../test/functional/BadIteratorMincIT.java      |  6 +--
 .../test/functional/BatchScanSplitIT.java       |  7 ++--
 .../test/functional/BatchWriterFlushIT.java     |  5 ++-
 .../accumulo/test/functional/BulkFileIT.java    |  5 ++-
 .../accumulo/test/functional/ConcurrencyIT.java |  7 ++--
 .../accumulo/test/functional/ConstraintIT.java  | 23 ++++++-----
 .../test/functional/CreateAndUseIT.java         |  5 ++-
 .../test/functional/DeleteEverythingIT.java     |  4 +-
 .../test/functional/DeleteRowsSplitIT.java      |  6 +--
 .../accumulo/test/functional/LargeRowIT.java    |  7 ++--
 .../accumulo/test/functional/NativeMapIT.java   | 15 +++----
 .../test/functional/ScanIteratorIT.java         |  7 ++--
 .../accumulo/test/functional/ScanRangeIT.java   |  5 ++-
 .../test/functional/ScanSessionTimeOutIT.java   |  5 ++-
 .../test/functional/SplitRecoveryIT.java        |  3 +-
 .../accumulo/test/functional/TabletIT.java      |  4 +-
 .../accumulo/test/functional/VisibilityIT.java  | 11 ++---
 .../instrument/receivers/ZooSpanClient.java     |  4 +-
 215 files changed, 900 insertions(+), 667 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/assemble/bin/bootstrap_config.sh
----------------------------------------------------------------------
diff --cc assemble/bin/bootstrap_config.sh
index fb643da,0000000..fd262f0
mode 100755,000000..100755
--- a/assemble/bin/bootstrap_config.sh
+++ b/assemble/bin/bootstrap_config.sh
@@@ -1,323 -1,0 +1,323 @@@
 +#! /usr/bin/env bash
 +
 +# Licensed to the Apache Software Foundation (ASF) under one or more
 +# contributor license agreements.  See the NOTICE file distributed with
 +# this work for additional information regarding copyright ownership.
 +# The ASF licenses this file to You under the Apache License, Version 2.0
 +# (the "License"); you may not use this file except in compliance with
 +# the License.  You may obtain a copy of the License at
 +#
 +#     http://www.apache.org/licenses/LICENSE-2.0
 +#
 +# Unless required by applicable law or agreed to in writing, software
 +# distributed under the License is distributed on an "AS IS" BASIS,
 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 +# See the License for the specific language governing permissions and
 +# limitations under the License.
 +
 +function usage {
 +  cat <<EOF
 +Usage: bootstrap_config.sh [-options]
 +where options include (long options not available on all platforms):
 +    -d, --dir        Alternate directory to setup config files
 +    -s, --size       Supported sizes: '1GB' '2GB' '3GB' '512MB'
 +    -n, --native     Configure to use native libraries
 +    -j, --jvm        Configure to use the jvm
 +    -o, --overwrite  Overwrite the default config directory
 +    -v, --version    Specify the Apache Hadoop version supported versions: '1' '2'
 +    -h, --help       Print this help message
 +EOF
 +}
 +
 +# Start: Resolve Script Directory
 +SOURCE="${BASH_SOURCE[0]}"
 +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
 +  bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 +  SOURCE="$(readlink "$SOURCE")"
 +  [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 +done
 +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 +
 +# Stop: Resolve Script Directory
 +
 +#
 +# Resolve accumulo home for bootstrapping
 +#
 +ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd )
 +TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates"
 +CONF_DIR="${ACCUMULO_HOME}/conf"
 +ACCUMULO_SITE=accumulo-site.xml
 +ACCUMULO_ENV=accumulo-env.sh
 +
 +SIZE=
 +TYPE=
 +HADOOP_VERSION=
 +OVERWRITE="0"
 +BASE_DIR=
 +
 +#Execute getopt
 +if [[ $(uname -s) == "Linux" ]]; then
 +  args=$(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@")
 +else # Darwin, BSD
 +  args=$(getopt b:d:s:njov:h $*)
 +fi
 +
 +#Bad arguments
 +if [[ $? != 0 ]]; then
 +  usage 1>&2
 +  exit 1
 +fi
 +eval set -- $args
 +
 +for i
 +do
 +  case "$i" in
 +    -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers
 +      BASE_DIR=$2; shift
 +      shift;;
 +    -d|--dir)
 +      CONF_DIR=$2; shift
 +      shift;;
 +    -s|--size)
 +      SIZE=$2; shift
 +      shift;;
 +    -n|--native)
 +      TYPE=native
 +      shift;;
 +    -j|--jvm)
 +      TYPE=jvm
 +      shift;;
 +    -o|--overwrite)
 +      OVERWRITE=1
 +      shift;;
 +    -v|--version)
 +      HADOOP_VERSION=$2; shift
 +      shift;;
 +    -h|--help)
 +      usage
 +      exit 0
 +      shift;;
 +    --)
 +    shift
 +    break;;
 +  esac
 +done
 +
 +while [[ "${OVERWRITE}" = "0" ]]; do
 +  if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then
 +    echo "Warning your current config files in ${CONF_DIR} will be overwritten!"
 +    echo
 +    echo "How would you like to proceed?:"
 +    select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do
 +      if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then
 +        echo -n "Please specifiy new conf directory: "
 +        read CONF_DIR
 +      elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then
 +        OVERWRITE=1
 +      fi
 +      break
 +    done
 +  else
 +    OVERWRITE=1
 +  fi
 +done
- echo "Coppying configuration files to: ${CONF_DIR}"
++echo "Copying configuration files to: ${CONF_DIR}"
 +
 +#Native 1GB
 +native_1GB_tServer="-Xmx128m -Xms128m"
 +_1GB_master="-Xmx128m -Xms128m"
 +_1GB_monitor="-Xmx64m -Xms64m"
 +_1GB_gc="-Xmx64m -Xms64m"
 +_1GB_other="-Xmx128m -Xms64m"
 +
 +_1GB_memoryMapMax="256M"
 +native_1GB_nativeEnabled="true"
 +_1GB_cacheDataSize="15M"
 +_1GB_cacheIndexSize="40M"
 +_1GB_sortBufferSize="50M"
 +_1GB_waLogMaxSize="256M"
 +
 +#Native 2GB
 +native_2GB_tServer="-Xmx256m -Xms256m"
 +_2GB_master="-Xmx256m -Xms256m"
 +_2GB_monitor="-Xmx128m -Xms64m"
 +_2GB_gc="-Xmx128m -Xms128m"
 +_2GB_other="-Xmx256m -Xms64m"
 +
 +_2GB_memoryMapMax="512M"
 +native_2GB_nativeEnabled="true"
 +_2GB_cacheDataSize="30M"
 +_2GB_cacheIndexSize="80M"
 +_2GB_sortBufferSize="50M"
 +_2GB_waLogMaxSize="512M"
 +
 +#Native 3GB
 +native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m"
 +_3GB_master="-Xmx1g -Xms1g"
 +_3GB_monitor="-Xmx1g -Xms256m"
 +_3GB_gc="-Xmx256m -Xms256m"
 +_3GB_other="-Xmx1g -Xms256m"
 +
 +_3GB_memoryMapMax="1G"
 +native_3GB_nativeEnabled="true"
 +_3GB_cacheDataSize="128M"
 +_3GB_cacheIndexSize="128M"
 +_3GB_sortBufferSize="200M"
 +_3GB_waLogMaxSize="1G"
 +
 +#Native 512MB
 +native_512MB_tServer="-Xmx48m -Xms48m"
 +_512MB_master="-Xmx128m -Xms128m"
 +_512MB_monitor="-Xmx64m -Xms64m"
 +_512MB_gc="-Xmx64m -Xms64m"
 +_512MB_other="-Xmx128m -Xms64m"
 +
 +_512MB_memoryMapMax="80M"
 +native_512MB_nativeEnabled="true"
 +_512MB_cacheDataSize="7M"
 +_512MB_cacheIndexSize="20M"
 +_512MB_sortBufferSize="50M"
 +_512MB_waLogMaxSize="100M"
 +
 +#JVM 1GB
 +jvm_1GB_tServer="-Xmx384m -Xms384m"
 +
 +jvm_1GB_nativeEnabled="false"
 +
 +#JVM 2GB
 +jvm_2GB_tServer="-Xmx768m -Xms768m"
 +
 +jvm_2GB_nativeEnabled="false"
 +
 +#JVM 3GB
 +jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G"
 +
 +jvm_3GB_nativeEnabled="false"
 +
 +#JVM 512MB
 +jvm_512MB_tServer="-Xmx128m -Xms128m"
 +
 +jvm_512MB_nativeEnabled="false"
 +
 +
 +if [[ -z "${SIZE}" ]]; then
 +  echo "Choose the heap configuration:"
 +  select DIRNAME in 1GB 2GB 3GB 512MB; do
 +    echo "Using '${DIRNAME}' configuration"
 +    SIZE=${DIRNAME}
 +    break
 +  done
 +elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB"  && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then
 +  echo "Invalid memory size"
 +  echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'"
 +  exit 1
 +fi
 +
 +if [[ -z "${TYPE}" ]]; then
 +  echo
 +  echo "Choose the Accumulo memory-map type:"
 +  select TYPENAME in Java Native; do
 +    if [[ "${TYPENAME}" == "Native" ]]; then
 +      TYPE="native"
 +      echo "Don't forget to build the native libraries using the bin/build_native_library.sh script"
 +    elif [[ "${TYPENAME}" == "Java" ]]; then
 +      TYPE="jvm"
 +    fi
 +    echo "Using '${TYPE}' configuration"
 +    echo
 +    break
 +  done
 +fi
 +
 +if [[ -z "${HADOOP_VERSION}" ]]; then
 +  echo
 +  echo "Choose the Apache Hadoop version:"
 +  select HADOOP in 'HADOOP 1' 'HADOOP 2' ; do
 +    if [ "${HADOOP}" == "HADOOP 2" ]; then
 +      HADOOP_VERSION="2"
 +    elif [ "${HADOOP}" == "HADOOP 1" ]; then
 +      HADOOP_VERSION="1"
 +    fi
 +    echo "Using Apache Hadoop version '${HADOOP_VERSION}' configuration"
 +    echo
 +    break
 +  done
 +elif [[ "${HADOOP_VERSION}" != "1" && "${HADOOP_VERSION}" != "2" ]]; then
 +  echo "Invalid Apache Hadoop version"
 +  echo "Supported Apache Hadoop versions: '1' '2'"
 +  exit 1
 +fi
 +
 +for var in SIZE TYPE HADOOP_VERSION; do
 +  if [[ -z ${!var} ]]; then
 +    echo "Invalid $var configuration"
 +    exit 1
 +  fi
 +done
 +
 +TSERVER="${TYPE}_${SIZE}_tServer"
 +MASTER="_${SIZE}_master"
 +MONITOR="_${SIZE}_monitor"
 +GC="_${SIZE}_gc"
 +OTHER="_${SIZE}_other"
 +
 +MEMORY_MAP_MAX="_${SIZE}_memoryMapMax"
 +NATIVE="${TYPE}_${SIZE}_nativeEnabled"
 +CACHE_DATA_SIZE="_${SIZE}_cacheDataSize"
 +CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize"
 +SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize"
 +WAL_MAX_SIZE="_${SIZE}_waLogMaxSize"
 +
 +MAVEN_PROJ_BASEDIR=""
 +
 +if [[ ! -z "${BASE_DIR}" ]]; then
 +  MAVEN_PROJ_BASEDIR="\n  <property>\n    <name>general.maven.project.basedir</name>\n    <value>${BASE_DIR}</value>\n  </property>\n"
 +fi
 +
 +#Configure accumulo-env.sh
 +mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/
 +sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \
 +    -e "s/\${masterHigh_masterLow}/${!MASTER}/" \
 +    -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \
 +    -e "s/\${gcHigh_gcLow}/${!GC}/" \
 +    -e "s/\${otherHigh_otherLow}/${!OTHER}/" \
 +    ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV
 +
 +#Configure accumulo-site.xml
 +sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \
 +    -e "s/\${nativeEnabled}/${!NATIVE}/" \
 +    -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \
 +    -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \
 +    -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \
 +    -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \
 +    -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE
 +
 +#Configure for hadoop 1
 +if [[ "$HADOOP_VERSION" = "1" ]]; then
 +  sed -e 's/^test -z \"$HADOOP_CONF_DIR\"/#test -z \"$HADOOP_CONF_DIR\"/' -e 's/^# test -z "$HADOOP_CONF_DIR"/test -z \"$HADOOP_CONF_DIR\"/' ${CONF_DIR}/$ACCUMULO_ENV > temp
 +  mv temp ${CONF_DIR}/$ACCUMULO_ENV
 +  sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements -->\n      <!--/' \
 +      -e 's/<!-- End Hadoop 2 requirements -->/-->\n      <!-- End Hadoop 2 requirements -->/' \
 +      ${CONF_DIR}/$ACCUMULO_SITE > temp
 +  mv temp  ${CONF_DIR}/$ACCUMULO_SITE
 +fi
 +
 +#Additional setup steps for native configuration.
 +if [[ "${TYPE}" = "native" ]]; then
 +  if [[ "$(uname)" = 'Linux' ]]; then
 +    if [[ -z $HADOOP_PREFIX ]]; then
 +      echo "HADOOP_PREFIX not set cannot automatically configure LD_LIBRARY_PATH"
 +    else
 +      NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null)
 +      if [[ -z $NATIVE_LIB ]]; then
 +        echo -e "Native libraries could not be found for your sytem in: $HADOOP_PREFIX"
 +      else
 +        sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp
 +        mv temp ${CONF_DIR}/$ACCUMULO_ENV
 +        echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH"
 +      fi
 +    fi
 +  fi
 +  echo -e "Please remember to compile the native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed."
 +fi
 +echo "Setup complete"

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 2a570f4,240de73..9897262
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -16,17 -16,37 +16,26 @@@
   */
  package org.apache.accumulo.core;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.nio.charset.Charset;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
  
 -import org.apache.accumulo.core.conf.AccumuloConfiguration;
 -import org.apache.accumulo.core.conf.Property;
 -import org.apache.accumulo.core.data.Key;
 -import org.apache.accumulo.core.data.KeyExtent;
 -import org.apache.accumulo.core.data.PartialKey;
 -import org.apache.accumulo.core.data.Range;
  import org.apache.accumulo.core.security.Authorizations;
 -import org.apache.accumulo.core.util.ColumnFQ;
 -import org.apache.hadoop.fs.Path;
 -import org.apache.hadoop.io.Text;
  
+ import com.google.common.base.Charsets;
+ 
  public class Constants {
-   public static final Charset UTF8 = Charset.forName("UTF-8");
+   /**
 -   * @deprecated since 1.5.3; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
++   * @deprecated since 1.5.3, 1.6.2; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
+    *             {@link Charset#forName(String)} with "UTF-8" instead
+    */
+   @Deprecated
+   public static final Charset UTF8 = UTF_8;
    public static final String VERSION = FilteredConstants.VERSION;
 -  
 -  // versions should never be negative
 -  public static final Integer WIRE_VERSION = 2;
 -  public static final int DATA_VERSION = 5;
 -  public static final int PREV_DATA_VERSION = 4;
 -  
 +
    // Zookeeper locations
    public static final String ZROOT = "/accumulo";
    public static final String ZINSTANCES = "/instances";
@@@ -56,62 -76,117 +65,62 @@@
    public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
    public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
    public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
 -  
 +
    public static final String ZCONFIG = "/config";
 -  
 +
    public static final String ZTSERVERS = "/tservers";
 -  
 +
    public static final String ZDEAD = "/dead";
 -  public static final String ZDEADTSERVERS = "/dead/tservers";
 -  
 +  public static final String ZDEADTSERVERS = ZDEAD + "/tservers";
 +
    public static final String ZTRACERS = "/tracers";
 -  
 +
    public static final String ZPROBLEMS = "/problems";
 -  public static final String ZUSERS = "/users";
 -  
 +
    public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
 -  
 +
    public static final String ZFATE = "/fate";
 -  
 +
    public static final String ZNEXT_FILE = "/next_file";
 -  
 +
    public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
 -  
 +
    public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
    public static final String ZRECOVERY = "/recovery";
 -  
 -  public static final String METADATA_TABLE_ID = "!0";
 -  public static final String METADATA_TABLE_NAME = "!METADATA";
 -  public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
 -  public static final String TABLE_TABLET_LOCATION = "/table_info";
 -  public static final String ZTABLE_LOCKS = "/table_locks";
 -  
 -  // reserved keyspace is any row that begins with a tilde '~' character
 -  public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
 -  public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
 -  public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
 -  public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
 -  public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
 -  public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
 -  public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
 -  public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
 -  public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
 -  
 -  public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
 -  public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
 -  public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
 -  public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
 -  public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
 -  public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
 -  public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
 -  
 -  // README : very important that prevRow sort last to avoid race conditions between
 -  // garbage collector and split
 -  public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
 -                                                                                                                        // else for that tablet
 -  public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
 -  public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
 -  public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
 -  public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
 -  public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
 -  public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
 -  public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
 -  
 -  public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
 -  public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
 -  public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
 -  public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
 -  public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
 -  
 -  public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
 -      new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
 -  public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
 -  
 -  public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
 -      null);
 -  public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
 -      METADATA_TABLE_ID), null), true);
 -  
 +
    /**
 -   * @deprecated since 1.5.3; not used, but specifies UTF-8
 +   * Initial tablet directory name for the default tablet in all tables
     */
 -  @Deprecated
 -  public static final String VALUE_ENCODING = UTF_8.name();
 -  
 +  public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
 +
 +  public static final String ZTABLE_LOCKS = "/table_locks";
 +
    public static final String BULK_PREFIX = "b-";
 -  public static final String OLD_BULK_PREFIX = "bulk_";
 -  
 -  // note: all times are in milliseconds
 -  
 -  public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
 -  
 -  public static final long MIN_MASTER_LOOP_TIME = 1000;
 -  public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
 -  public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
 -  
 +
 +  public static final String CLONE_PREFIX = "c-";
-   public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF8);
++  public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF_8);
 +
 +  // this affects the table client caching of metadata
 +  public static final int SCAN_BATCH_SIZE = 1000;
 +
 +  // Scanners will default to fetching 3 batches of Key/Value pairs before asynchronously
 +  // fetching the next batch.
 +  public static final long SCANNER_DEFAULT_READAHEAD_THRESHOLD = 3l;
 +
    // Security configuration
    public static final String PW_HASH_ALGORITHM = "SHA-256";
 -  
 -  // Representation of an empty set of authorizations
 -  // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
 -  public static final Authorizations NO_AUTHS = new Authorizations();
 -  
 -  public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
 -  
 +
 +  /**
 +   * @deprecated since 1.6.0; Use {@link Authorizations#EMPTY} instead
 +   */
 +  @Deprecated
 +  public static final Authorizations NO_AUTHS = Authorizations.EMPTY;
 +
    public static final int MAX_DATA_TO_PRINT = 64;
 -  public static final int CLIENT_RETRIES = 5;
 -  public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
    public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
 -  public static final String OLD_PACKAGE_NAME = "cloudbase";
 -  public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
    public static final String MAPFILE_EXTENSION = "map";
    public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
 -  
 +
    public static final String EXPORT_METADATA_FILE = "metadata.bin";
    public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
    public static final String EXPORT_FILE = "exportMetadata.zip";

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
index 35a9067,5134521..041c9e6
--- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
@@@ -16,6 -16,9 +16,8 @@@
   */
  package org.apache.accumulo.core.cli;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import java.util.Iterator;
  import java.util.LinkedHashMap;
  import java.util.Map;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 6d35757,251d6a7..a9c3292
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -16,6 -16,11 +16,8 @@@
   */
  package org.apache.accumulo.core.client;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import java.io.FileNotFoundException;
 -import java.io.IOException;
 -import java.net.UnknownHostException;
  import java.nio.ByteBuffer;
  import java.util.Collections;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
index 63e3497,0000000..6925f18
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
@@@ -1,210 -1,0 +1,212 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.ActiveScan;
 +import org.apache.accumulo.core.client.admin.ActiveCompaction;
 +import org.apache.accumulo.core.client.admin.InstanceOperations;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService;
 +import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +
 +/**
 + * Provides a class for administering the accumulo instance
 + */
 +public class InstanceOperationsImpl implements InstanceOperations {
 +  private Instance instance;
 +  private Credentials credentials;
 +
 +  /**
 +   * @param instance
 +   *          the connection information for this instance
 +   * @param credentials
 +   *          the Credential, containing principal and Authentication Token
 +   */
 +  public InstanceOperationsImpl(Instance instance, Credentials credentials) {
 +    ArgumentChecker.notNull(instance, credentials);
 +    this.instance = instance;
 +    this.credentials = credentials;
 +  }
 +
 +  @Override
 +  public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(property, value);
 +    MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
 +      @Override
 +      public void execute(MasterClientService.Client client) throws Exception {
 +        client.setSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property, value);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(property);
 +    MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
 +      @Override
 +      public void execute(MasterClientService.Client client) throws Exception {
 +        client.removeSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
 +    return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
 +      @Override
 +      public Map<String,String> execute(ClientService.Client client) throws Exception {
 +        return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.CURRENT);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
 +    return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
 +      @Override
 +      public Map<String,String> execute(ClientService.Client client) throws Exception {
 +        return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.SITE);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public List<String> getTabletServers() {
 +    ZooCache cache = new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +    List<String> results = new ArrayList<String>();
 +    for (String candidate : cache.getChildren(path)) {
 +      List<String> children = cache.getChildren(path + "/" + candidate);
 +      if (children != null && children.size() > 0) {
 +        List<String> copy = new ArrayList<String>(children);
 +        Collections.sort(copy);
 +        byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
-         if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
++        if (data != null && !"master".equals(new String(data, UTF_8))) {
 +          results.add(candidate);
 +        }
 +      }
 +    }
 +    return results;
 +  }
 +
 +  @Override
 +  public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
 +    Client client = null;
 +    try {
 +      client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
 +
 +      List<ActiveScan> as = new ArrayList<ActiveScan>();
 +      for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials.toThrift(instance))) {
 +        try {
 +          as.add(new ActiveScanImpl(instance, activeScan));
 +        } catch (TableNotFoundException e) {
 +          throw new AccumuloException(e);
 +        }
 +      }
 +      return as;
 +    } catch (TTransportException e) {
 +      throw new AccumuloException(e);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (TException e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (client != null)
 +        ThriftUtil.returnClient(client);
 +    }
 +  }
 +
 +  @Override
 +  public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
 +    return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
 +      @Override
 +      public Boolean execute(ClientService.Client client) throws Exception {
 +        return client.checkClass(Tracer.traceInfo(), credentials.toThrift(instance), className, asTypeName);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
 +    Client client = null;
 +    try {
 +      client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
 +
 +      List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
 +      for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(),
 +          credentials.toThrift(instance))) {
 +        as.add(new ActiveCompactionImpl(instance, activeCompaction));
 +      }
 +      return as;
 +    } catch (TTransportException e) {
 +      throw new AccumuloException(e);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (TException e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (client != null)
 +        ThriftUtil.returnClient(client);
 +    }
 +  }
 +
 +  @Override
 +  public void ping(String tserver) throws AccumuloException {
 +    TTransport transport = null;
 +    try {
 +      transport = ThriftUtil.createTransport(AddressUtil.parseAddress(tserver, false), ServerConfigurationUtil.getConfiguration(instance));
 +      TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
 +      client.getTabletServerStatus(Tracer.traceInfo(), credentials.toThrift(instance));
 +    } catch (TTransportException e) {
 +      throw new AccumuloException(e);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloException(e);
 +    } catch (TException e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (transport != null) {
 +        transport.close();
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
index 30f0266,0000000..f32940c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
@@@ -1,154 -1,0 +1,156 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.security.SecurityPermission;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.util.ArgumentChecker.Validator;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +
 +public class Namespaces {
 +  public static final String VALID_NAME_REGEX = "^\\w*$";
 +  public static final Validator<String> VALID_NAME = new Validator<String>() {
 +    @Override
 +    public boolean isValid(String namespace) {
 +      return namespace != null && namespace.matches(VALID_NAME_REGEX);
 +    }
 +
 +    @Override
 +    public String invalidMessage(String namespace) {
 +      if (namespace == null)
 +        return "Namespace cannot be null";
 +      return "Namespaces must only contain word characters (letters, digits, and underscores): " + namespace;
 +    }
 +  };
 +
 +  public static final Validator<String> NOT_DEFAULT = new Validator<String>() {
 +    @Override
 +    public boolean isValid(String namespace) {
 +      return !Namespaces.DEFAULT_NAMESPACE.equals(namespace);
 +    }
 +
 +    @Override
 +    public String invalidMessage(String namespace) {
 +      return "Namespace cannot be the reserved empty namespace";
 +    }
 +  };
 +
 +  public static final Validator<String> NOT_ACCUMULO = new Validator<String>() {
 +    @Override
 +    public boolean isValid(String namespace) {
 +      return !Namespaces.ACCUMULO_NAMESPACE.equals(namespace);
 +    }
 +
 +    @Override
 +    public String invalidMessage(String namespace) {
 +      return "Namespace cannot be the reserved namespace, " + Namespaces.ACCUMULO_NAMESPACE;
 +    }
 +  };
 +
 +  private static SecurityPermission TABLES_PERMISSION = new SecurityPermission("tablesPermission");
 +
 +  public static final String DEFAULT_NAMESPACE_ID = "+default";
 +  public static final String DEFAULT_NAMESPACE = "";
 +  public static final String ACCUMULO_NAMESPACE_ID = "+accumulo";
 +  public static final String ACCUMULO_NAMESPACE = "accumulo";
 +
 +  private static ZooCache getZooCache(Instance instance) {
 +    SecurityManager sm = System.getSecurityManager();
 +    if (sm != null) {
 +      sm.checkPermission(TABLES_PERMISSION);
 +    }
 +    return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +  }
 +
 +  private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
 +    ZooCache zc = getZooCache(instance);
 +
 +    List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
 +
 +    TreeMap<String,String> namespaceMap = new TreeMap<String,String>();
 +
 +    for (String id : namespaceIds) {
 +      byte[] path = zc.get(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + id + Constants.ZNAMESPACE_NAME);
 +      if (path != null) {
 +        if (nameAsKey)
-           namespaceMap.put(new String(path, Constants.UTF8), id);
++          namespaceMap.put(new String(path, UTF_8), id);
 +        else
-           namespaceMap.put(id, new String(path, Constants.UTF8));
++          namespaceMap.put(id, new String(path, UTF_8));
 +      }
 +    }
 +    return namespaceMap;
 +  }
 +
 +  public static boolean exists(Instance instance, String namespaceId) {
 +    ZooCache zc = getZooCache(instance);
 +    List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
 +    return namespaceIds.contains(namespaceId);
 +  }
 +
 +  public static String getNamespaceId(Instance instance, String namespace) throws NamespaceNotFoundException {
 +    String id = getNameToIdMap(instance).get(namespace);
 +    if (id == null)
 +      throw new NamespaceNotFoundException(null, namespace, "getNamespaceId() failed to find namespace");
 +    return id;
 +  }
 +
 +  public static String getNamespaceName(Instance instance, String namespaceId) throws NamespaceNotFoundException {
 +    String namespaceName = getIdToNameMap(instance).get(namespaceId);
 +    if (namespaceName == null)
 +      throw new NamespaceNotFoundException(namespaceId, null, "getNamespaceName() failed to find namespace");
 +    return namespaceName;
 +  }
 +
 +  public static SortedMap<String,String> getNameToIdMap(Instance instance) {
 +    return getMap(instance, true);
 +  }
 +
 +  public static SortedMap<String,String> getIdToNameMap(Instance instance) {
 +    return getMap(instance, false);
 +  }
 +
 +  public static List<String> getTableIds(Instance instance, String namespaceId) throws NamespaceNotFoundException {
 +    String namespace = getNamespaceName(instance, namespaceId);
 +    List<String> names = new LinkedList<String>();
 +    for (Entry<String,String> nameToId : Tables.getNameToIdMap(instance).entrySet())
 +      if (namespace.equals(Tables.qualify(nameToId.getKey()).getFirst()))
 +        names.add(nameToId.getValue());
 +    return names;
 +  }
 +
 +  public static List<String> getTableNames(Instance instance, String namespaceId) throws NamespaceNotFoundException {
 +    String namespace = getNamespaceName(instance, namespaceId);
 +    List<String> names = new LinkedList<String>();
 +    for (String name : Tables.getNameToIdMap(instance).keySet())
 +      if (namespace.equals(Tables.qualify(name).getFirst()))
 +        names.add(name);
 +    return names;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
index 8da530b,1fcebe7..70bdbb0
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
@@@ -16,7 -16,11 +16,9 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.util.ArrayList;
 -import java.util.HashMap;
 -import java.util.Map;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.AccumuloException;
@@@ -128,10 -142,13 +130,10 @@@ public class ServerClient 
      for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
        String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
        byte[] data = ZooUtil.getLockData(zc, path);
-       if (data != null && !new String(data, Constants.UTF8).equals("master"))
 -      if (data != null) {
 -        String tserverData = new String(data, UTF_8);
 -        if (!tserverData.equals("master")) {
 -          servers.add(new ThriftTransportKey(new ServerServices(tserverData).getAddressString(Service.TSERV_CLIENT), conf.getPort(Property.TSERV_CLIENTPORT),
 -              rpcTimeout));
 -        }
 -      }
++      if (data != null && !new String(data, UTF_8).equals("master"))
 +        servers.add(new ThriftTransportKey(
 +          new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT),
 +          rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance))));
      }
      
      boolean opened = false;


[39/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index bfd4818,0000000..e6e31eb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -1,238 -1,0 +1,239 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Arrays;
 +import java.util.HashSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.hadoop.io.Text;
 +
 +/**
 + * Conditions that must be met on a particular column in a row.
 + * 
 + * @since 1.6.0
 + */
 +public class Condition {
 +  
 +  private ByteSequence cf;
 +  private ByteSequence cq;
 +  private ByteSequence cv;
 +  private ByteSequence val;
 +  private Long ts;
 +  private IteratorSetting iterators[] = new IteratorSetting[0];
 +  private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
 +  
 +
 +  public Condition(CharSequence cf, CharSequence cq) {
 +    ArgumentChecker.notNull(cf, cq);
-     this.cf = new ArrayByteSequence(cf.toString().getBytes(Constants.UTF8));
-     this.cq = new ArrayByteSequence(cq.toString().getBytes(Constants.UTF8));
++    this.cf = new ArrayByteSequence(cf.toString().getBytes(UTF_8));
++    this.cq = new ArrayByteSequence(cq.toString().getBytes(UTF_8));
 +    this.cv = EMPTY;
 +  }
 +  
 +  public Condition(byte[] cf, byte[] cq) {
 +    ArgumentChecker.notNull(cf, cq);
 +    this.cf = new ArrayByteSequence(cf);
 +    this.cq = new ArrayByteSequence(cq);
 +    this.cv = EMPTY;
 +  }
 +
 +  public Condition(Text cf, Text cq) {
 +    ArgumentChecker.notNull(cf, cq);
 +    this.cf = new ArrayByteSequence(cf.getBytes(), 0, cf.getLength());
 +    this.cq = new ArrayByteSequence(cq.getBytes(), 0, cq.getLength());
 +    this.cv = EMPTY;
 +  }
 +
 +  public Condition(ByteSequence cf, ByteSequence cq) {
 +    ArgumentChecker.notNull(cf, cq);
 +    this.cf = cf;
 +    this.cq = cq;
 +    this.cv = EMPTY;
 +  }
 +
 +  public ByteSequence getFamily() {
 +    return cf;
 +  }
 +  
 +  public ByteSequence getQualifier() {
 +    return cq;
 +  }
 +
 +  /**
 +   * Sets the version for the column to check. If this is not set then the latest column will be checked, unless iterators do something different.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setTimestamp(long ts) {
 +    this.ts = ts;
 +    return this;
 +  }
 +  
 +  public Long getTimestamp() {
 +    return ts;
 +  }
 +
 +  /**
 +   * see {@link #setValue(byte[])}
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(CharSequence value) {
 +    ArgumentChecker.notNull(value);
-     this.val = new ArrayByteSequence(value.toString().getBytes(Constants.UTF8));
++    this.val = new ArrayByteSequence(value.toString().getBytes(UTF_8));
 +    return this;
 +  }
 +
 +  /**
 +   * This method sets the expected value of a column. Inorder for the condition to pass the column must exist and have this value. If a value is not set, then
 +   * the column must be absent for the condition to pass.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(byte[] value) {
 +    ArgumentChecker.notNull(value);
 +    this.val = new ArrayByteSequence(value);
 +    return this;
 +  }
 +  
 +  /**
 +   * see {@link #setValue(byte[])}
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(Text value) {
 +    ArgumentChecker.notNull(value);
 +    this.val = new ArrayByteSequence(value.getBytes(), 0, value.getLength());
 +    return this;
 +  }
 +  
 +  /**
 +   * see {@link #setValue(byte[])}
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(ByteSequence value) {
 +    ArgumentChecker.notNull(value);
 +    this.val = value;
 +    return this;
 +  }
 +
 +  public ByteSequence getValue() {
 +    return val;
 +  }
 +
 +  /**
 +   * Sets the visibility for the column to check. If not set it defaults to empty visibility.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setVisibility(ColumnVisibility cv) {
 +    ArgumentChecker.notNull(cv);
 +    this.cv = new ArrayByteSequence(cv.getExpression());
 +    return this;
 +  }
 +
 +  public ByteSequence getVisibility() {
 +    return cv;
 +  }
 +
 +  /**
 +   * Set iterators to use when reading the columns value. These iterators will be applied in addition to the iterators configured for the table. Using iterators
 +   * its possible to test other conditions, besides equality and absence, like less than. On the server side the iterators will be seeked using a range that
 +   * covers only the family, qualifier, and visibility (if the timestamp is set then it will be used to narrow the range). Value equality will be tested using
 +   * the first entry returned by the iterator stack.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setIterators(IteratorSetting... iterators) {
 +    ArgumentChecker.notNull(iterators);
 +    
 +    if (iterators.length > 1) {
 +      HashSet<String> names = new HashSet<String>();
 +      HashSet<Integer> prios = new HashSet<Integer>();
 +      
 +      for (IteratorSetting iteratorSetting : iterators) {
 +        if (!names.add(iteratorSetting.getName()))
 +          throw new IllegalArgumentException("iterator name used more than once " + iteratorSetting.getName());
 +        if (!prios.add(iteratorSetting.getPriority()))
 +          throw new IllegalArgumentException("iterator priority used more than once " + iteratorSetting.getPriority());
 +      }
 +    }
 +    
 +    this.iterators = iterators;
 +    return this;
 +  }
 +
 +  public IteratorSetting[] getIterators() {
 +    return iterators;
 +  }
 +
 +  @Override
 +  public boolean equals(Object o) {
 +    if (o == this) {
 +      return true;
 +    }
 +    if (o == null || !(o instanceof Condition)) {
 +      return false;
 +    }
 +    Condition c = (Condition) o;
 +    if (!(c.cf.equals(cf))) {
 +      return false;
 +    }
 +    if (!(c.cq.equals(cq))) {
 +      return false;
 +    }
 +    if (!(c.cv.equals(cv))) {
 +      return false;
 +    }
 +    if (!(c.val == null ? val == null : c.val.equals(val))) {
 +      return false;
 +    }
 +    if (!(c.ts == null ? ts == null : c.ts.equals(ts))) {
 +      return false;
 +    }
 +    if (!(Arrays.equals(c.iterators, iterators))) {
 +      return false;
 +    }
 +    return true;
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    int result = 17;
 +    result = 31 * result + cf.hashCode();
 +    result = 31 * result + cq.hashCode();
 +    result = 31 * result + cv.hashCode();
 +    result = 31 * result + (val == null ? 0 : val.hashCode());
 +    result = 31 * result + (ts == null ? 0 : ts.hashCode());
 +    result = 31 * result + Arrays.hashCode(iterators);
 +    return result;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Value.java
index 11e60e1,bc1f71d..d256107
--- a/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@@ -25,9 -26,6 +26,8 @@@ import java.io.IOException
  import java.nio.ByteBuffer;
  import java.util.List;
  
 +import com.google.common.base.Preconditions;
 +
- import org.apache.accumulo.core.Constants;
  import org.apache.hadoop.io.BytesWritable;
  import org.apache.hadoop.io.WritableComparable;
  import org.apache.hadoop.io.WritableComparator;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
index 8006114,0b220ee..7073d11
--- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
@@@ -24,10 -26,7 +26,9 @@@ import java.util.Collections
  import java.util.HashSet;
  import java.util.Iterator;
  import java.util.List;
 +import java.util.Set;
 +import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.data.ArrayByteSequence;
  import org.apache.accumulo.core.data.ByteSequence;
  import org.apache.accumulo.core.util.ArgumentChecker;
@@@ -195,13 -166,12 +196,13 @@@ public class Authorizations implements 
    }
  
    /**
 -   * Retrieve a serialized form of the underlying set of authorizations.
 +   * Returns a serialized form of these authorizations.
     * 
 -   * @see #Authorizations(byte[])
 +   * @return serialized form of these authorizations, as a string encoded in UTF-8
 +   * @see #serialize()
     */
    public byte[] getAuthorizationsArray() {
-     return serialize().getBytes(Constants.UTF8);
+     return serialize().getBytes(UTF_8);
    }
  
    /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index e76b0ef,66b68ab..e8f4e97
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -243,9 -223,9 +244,9 @@@ public class ColumnVisibility 
    /*
     * Walks an expression's AST and appends a string representation to a supplied StringBuilder. This method adds parens where necessary.
     */
 -  private static void stringify(Node root, byte[] expression, StringBuilder out) {
 +  public static void stringify(Node root, byte[] expression, StringBuilder out) {
      if (root.type == NodeType.TERM) {
-       out.append(new String(expression, root.start, root.end - root.start, Constants.UTF8));
+       out.append(new String(expression, root.start, root.end - root.start, UTF_8));
      } else {
        String sep = "";
        for (Node c : root.children) {
@@@ -314,12 -293,12 +315,12 @@@
        while (index < expression.length) {
          switch (expression[index++]) {
            case '&': {
 -            expr = processTerm(termStart, index - 1, expr, expression);
 +            expr = processTerm(subtermStart, index - 1, expr, expression);
              if (result != null) {
                if (!result.type.equals(NodeType.AND))
-                 throw new BadArgumentException("cannot mix & and |", new String(expression, Constants.UTF8), index - 1);
+                 throw new BadArgumentException("cannot mix & and |", new String(expression, UTF_8), index - 1);
              } else {
 -              result = new Node(NodeType.AND);
 +              result = new Node(NodeType.AND, wholeTermStart);
              }
              result.add(expr);
              expr = null;
@@@ -328,12 -307,12 +329,12 @@@
              break;
            }
            case '|': {
 -            expr = processTerm(termStart, index - 1, expr, expression);
 +            expr = processTerm(subtermStart, index - 1, expr, expression);
              if (result != null) {
                if (!result.type.equals(NodeType.OR))
-                 throw new BadArgumentException("cannot mix | and &", new String(expression, Constants.UTF8), index - 1);
+                 throw new BadArgumentException("cannot mix | and &", new String(expression, UTF_8), index - 1);
              } else {
 -              result = new Node(NodeType.OR);
 +              result = new Node(NodeType.OR, wholeTermStart);
              }
              result.add(expr);
              expr = null;
@@@ -343,18 -322,18 +344,18 @@@
            }
            case '(': {
              parens++;
 -            if (termStart != index - 1 || expr != null)
 +            if (subtermStart != index - 1 || expr != null)
-               throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+               throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
              expr = parse_(expression);
 -            termStart = index;
 -            termComplete = false;
 +            subtermStart = index;
 +            subtermComplete = false;
              break;
            }
            case ')': {
              parens--;
 -            Node child = processTerm(termStart, index - 1, expr, expression);
 +            Node child = processTerm(subtermStart, index - 1, expr, expression);
              if (child == null && result == null)
-               throw new BadArgumentException("empty expression not allowed", new String(expression, Constants.UTF8), index);
+               throw new BadArgumentException("empty expression not allowed", new String(expression, UTF_8), index);
              if (result == null)
                return child;
              if (result.type == child.type)
@@@ -366,8 -345,8 +367,8 @@@
              return result;
            }
            case '"': {
 -            if (termStart != index - 1)
 +            if (subtermStart != index - 1)
-               throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+               throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
              
              while (index < expression.length && expression[index] != '"') {
                if (expression[index] == '\\') {
@@@ -379,20 -358,20 +380,20 @@@
              }
              
              if (index == expression.length)
-               throw new BadArgumentException("unclosed quote", new String(expression, Constants.UTF8), subtermStart);
 -              throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), termStart);
 -            
 -            if (termStart + 1 == index)
 -              throw new BadArgumentException("empty term", new String(expression, UTF_8), termStart);
++              throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), subtermStart);
              
 +            if (subtermStart + 1 == index)
-               throw new BadArgumentException("empty term", new String(expression, Constants.UTF8), subtermStart);
++              throw new BadArgumentException("empty term", new String(expression, UTF_8), subtermStart);
 + 
              index++;
              
 -            termComplete = true;
 +            subtermComplete = true;
              
              break;
            }
            default: {
 -            if (termComplete)
 +            if (subtermComplete)
-               throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+               throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
              
              byte c = expression[index - 1];
              if (!Authorizations.isValidAuthChar(c))
@@@ -514,12 -482,10 +515,12 @@@
     *   .
     * ColumnVisibility cv = new ColumnVisibility(quote(&quot;A#C&quot;) + &quot;&amp;&quot; + quote(&quot;FOO&quot;));
     * </pre>
 -   * 
 +   *
 +   * @param term term to quote
 +   * @return quoted term (unquoted if unnecessary)
     */
    public static String quote(String term) {
-     return new String(quote(term.getBytes(Constants.UTF8)), Constants.UTF8);
+     return new String(quote(term.getBytes(UTF_8)), UTF_8);
    }
    
    /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Credentials.java
index 582b4e0,31fe18d..0ecc31d
--- a/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
@@@ -16,17 -16,10 +16,18 @@@
   */
  package org.apache.accumulo.core.security;
  
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.nio.ByteBuffer;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
  import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
  import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.util.Base64;
  
  /**
   * A wrapper for internal use. This class carries the instance, principal, and authentication token for use in the public API, in a non-serialized form. This is
@@@ -53,102 -36,9 +54,102 @@@ public class Credentials 
      this.principal = principal;
      this.token = token;
    }
 -  
 -  public TCredentials toThrift() {
 -    return CredentialHelper.createSquelchError(principal, token, instance.getInstanceID());
 +
 +  /**
 +   * Gets the principal.
 +   * 
 +   * @return unique identifier for the entity (e.g. a user or service) authorized for these credentials
 +   */
 +  public String getPrincipal() {
 +    return principal;
 +  }
 +
 +  /**
 +   * Gets the authentication token.
 +   * 
 +   * @return authentication token used to prove that the principal for these credentials has been properly verified
 +   */
 +  public AuthenticationToken getToken() {
 +    return token;
 +  }
 +
 +  /**
 +   * Converts the current object to the relevant thrift type. The object returned from this contains a non-destroyable version of the
 +   * {@link AuthenticationToken}, so this should be used just before placing on the wire, and references to it should be tightly controlled.
 +   * 
 +   * @param instance
 +   *          client instance
 +   * @return Thrift credentials
 +   * @throws RuntimeException
 +   *           if the authentication token has been destroyed (expired)
 +   */
 +  public TCredentials toThrift(Instance instance) {
 +    TCredentials tCreds = new TCredentials(getPrincipal(), getToken().getClass().getName(),
 +        ByteBuffer.wrap(AuthenticationTokenSerializer.serialize(getToken())), instance.getInstanceID());
 +    if (getToken().isDestroyed())
 +      throw new RuntimeException("Token has been destroyed", new AccumuloSecurityException(getPrincipal(), SecurityErrorCode.TOKEN_EXPIRED));
 +    return tCreds;
 +  }
 +
 +  /**
 +   * Converts a given thrift object to our internal Credentials representation.
 +   * @param serialized a Thrift encoded set of credentials
 +   * @return a new Credentials instance; destroy the token when you're done.
 +   */
 +  public static Credentials fromThrift(TCredentials serialized) {
 +    return new Credentials(serialized.getPrincipal(), AuthenticationTokenSerializer.deserialize(serialized.getTokenClassName(), serialized.getToken()));
 +  }
 +
 +  /**
 +   * Converts the current object to a serialized form. The object returned from this contains a non-destroyable version of the {@link AuthenticationToken}, so
 +   * references to it should be tightly controlled.
 +   * 
 +   * @return serialized form of these credentials
 +   */
 +  public final String serialize() {
-     return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(Constants.UTF8))) + ":"
-         + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(Constants.UTF8))) + ":"
++    return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(UTF_8))) + ":"
++        + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(UTF_8))) + ":"
 +        + (getToken() == null ? "-" : Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(getToken())));
 +  }
 +
 +  /**
 +   * Converts the serialized form to an instance of {@link Credentials}. The original serialized form will not be affected.
 +   * 
 +   * @param serializedForm
 +   *          serialized form of credentials
 +   * @return deserialized credentials
 +   */
 +  public static final Credentials deserialize(String serializedForm) {
 +    String[] split = serializedForm.split(":", 3);
-     String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), Constants.UTF8);
-     String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), Constants.UTF8);
++    String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), UTF_8);
++    String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), UTF_8);
 +    AuthenticationToken token = null;
 +    if (!split[2].equals("-")) {
 +      byte[] tokenBytes = Base64.decodeBase64(split[2]);
 +      token = AuthenticationTokenSerializer.deserialize(tokenType, tokenBytes);
 +    }
 +    return new Credentials(principal, token);
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    return getPrincipal() == null ? 0 : getPrincipal().hashCode();
 +  }
 +
 +  @Override
 +  public boolean equals(Object obj) {
 +    if (obj == null || !(obj instanceof Credentials))
 +      return false;
 +    Credentials other = Credentials.class.cast(obj);
 +    boolean pEq = getPrincipal() == null ? (other.getPrincipal() == null) : (getPrincipal().equals(other.getPrincipal()));
 +    if (!pEq)
 +      return false;
 +    boolean tEq = getToken() == null ? (other.getToken() == null) : (getToken().equals(other.getToken()));
 +    return tEq;
 +  }
 +
 +  @Override
 +  public String toString() {
 +    return getClass().getName() + ":" + getPrincipal() + ":" + (getToken() == null ? null : getToken().getClass().getName()) + ":<hidden>";
    }
 -  
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
index be5f008,9c5782b..178de91
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
@@@ -16,27 -16,17 +16,27 @@@
   */
  package org.apache.accumulo.core.security;
  
- import java.text.ParseException;
+ import static com.google.common.base.Charsets.UTF_8;
  
- import org.apache.accumulo.core.Constants;
+ import java.text.ParseException;
  
 +/**
 + * An exception thrown when a visibility string cannot be parsed.
 + */
  public class VisibilityParseException extends ParseException {
    private static final long serialVersionUID = 1L;
    private String visibility;
 -  
 +
 +  /**
 +   * Creates a new exception.
 +   *
 +   * @param reason reason string
 +   * @param visibility visibility that could not be parsed
 +   * @param errorOffset offset into visibility where parsing failed
 +   */
    public VisibilityParseException(String reason, byte[] visibility, int errorOffset) {
      super(reason, errorOffset);
-     this.visibility = new String(visibility, Constants.UTF8);
+     this.visibility = new String(visibility, UTF_8);
    }
    
    @Override

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
index 7a23c37,f6a6a01..8771832
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
@@@ -22,8 -24,6 +24,7 @@@ import java.util.Arrays
  import java.util.Collection;
  import java.util.List;
  
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.ByteSequence;
  import org.apache.hadoop.io.Text;
  
  public class ByteBufferUtil {
@@@ -62,18 -62,6 +63,18 @@@
    }
    
    public static String toString(ByteBuffer bytes) {
-     return new String(bytes.array(), bytes.position(), bytes.remaining(), Constants.UTF8);
+     return new String(bytes.array(), bytes.position(), bytes.remaining(), UTF_8);
    }
 +  
 +  public static ByteBuffer toByteBuffers(ByteSequence bs) {
 +    if (bs == null)
 +      return null;
 +
 +    if (bs.isBackedByArray()) {
 +      return ByteBuffer.wrap(bs.getBackingArray(), bs.offset(), bs.length());
 +    } else {
 +      // TODO create more efficient impl
 +      return ByteBuffer.wrap(bs.toArray());
 +    }
 +  }
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index 9f86db3,0000000..26a8c73
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@@ -1,117 -1,0 +1,118 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.io.IOException;
 +import java.io.PrintStream;
 +
 +import jline.console.ConsoleReader;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.ClientOpts.Password;
 +import org.apache.accumulo.core.cli.ClientOpts.PasswordConverter;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.util.Base64;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class CreateToken {
 +  
 +  private static ConsoleReader reader = null;
 +  
 +  private static ConsoleReader getConsoleReader() throws IOException {
 +    if (reader == null)
 +      reader = new ConsoleReader();
 +    return reader;
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = {"-u", "--user"}, description = "Connection user")
 +    public String principal = null;
 +    
 +    @Parameter(names = "-p", converter = PasswordConverter.class, description = "Connection password")
 +    public Password password = null;
 +    
 +    @Parameter(names = "--password", converter = PasswordConverter.class, description = "Enter the connection password", password = true)
 +    public Password securePassword = null;
 +    
 +    @Parameter(names = {"-tc", "--tokenClass"}, description = "The class of the authentication token")
 +    public String tokenClassName = PasswordToken.class.getName();
 +    
 +    @Parameter(names = {"-f", "--file"}, description = "The filename to save the auth token to. Multiple tokens can be stored in the same file,"
 +        + " but only the first for each user will be recognized.")
 +    public String tokenFile = null;
 +  }
 +  
 +  public static void main(String[] args) {
 +    Opts opts = new Opts();
 +    opts.parseArgs(CreateToken.class.getName(), args);
 +    
 +    Password pass = opts.password;
 +    if (pass == null && opts.securePassword != null) {
 +      pass = opts.securePassword;
 +    }
 +    
 +    try {
 +      String principal = opts.principal;
 +      if (principal == null) {
 +        principal = getConsoleReader().readLine("Username (aka principal): ");
 +      }
 +      
 +      AuthenticationToken token = Class.forName(opts.tokenClassName).asSubclass(AuthenticationToken.class).newInstance();
 +      Properties props = new Properties();
 +      for (TokenProperty tp : token.getProperties()) {
 +        String input;
 +        if (pass != null && tp.getKey().equals("password")) {
 +          input = pass.toString();
 +        } else {
 +          if (tp.getMask()) {
 +            input = getConsoleReader().readLine(tp.getDescription() + ": ", '*');
 +          } else {
 +            input = getConsoleReader().readLine(tp.getDescription() + ": ");
 +          }
 +        }
 +        props.put(tp.getKey(), input);
 +        token.init(props);
 +      }
 +      String tokenBase64 = Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token));
 +      
 +      String tokenFile = opts.tokenFile;
 +      if (tokenFile == null) {
 +        tokenFile = getConsoleReader().readLine("File to save auth token to: ");
 +      }
 +      File tf = new File(tokenFile);
 +      if (!tf.exists()) {
 +        tf.createNewFile();
 +      }
-       PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, Constants.UTF8.name());
++      PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, UTF_8.name());
 +      String outString = principal + ":" + opts.tokenClassName + ":" + tokenBase64;
 +      out.println(outString);
 +      out.close();
 +      System.out.println("Token written to " + tokenFile + ". Remember to upload it to hdfs.");
 +    } catch (Exception e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Encoding.java
index aff8f62,9999c3c..b26b363
--- a/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.util;
  
- import org.apache.accumulo.core.Constants;
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import org.apache.commons.codec.binary.Base64;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.hadoop.io.Text;
  
  public class Encoding {
@@@ -36,8 -38,10 +37,8 @@@
    public static byte[] decodeBase64FileName(String node) {
      while (node.length() % 4 != 0)
        node += "=";
 -    
 -    node = node.replace('_', '/').replace('-', '+');
 -    
 +    /* decode transparently handles URLSafe encodings */
-     return Base64.decodeBase64(node.getBytes(Constants.UTF8));
+     return Base64.decodeBase64(node.getBytes(UTF_8));
    }
    
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java
index 02c8be3,3830108..05067b5
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@@ -21,7 -23,7 +23,6 @@@ import java.util.Iterator
  import java.util.List;
  import java.util.Map.Entry;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.ClientOnRequiredTable;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Scanner;
@@@ -236,8 -233,8 +237,8 @@@ public class Merge 
          while (iterator.hasNext()) {
            Entry<Key,Value> entry = iterator.next();
            Key key = entry.getKey();
 -          if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
 +          if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
-             String[] sizeEntries = new String(entry.getValue().get(), Constants.UTF8).split(",");
+             String[] sizeEntries = new String(entry.getValue().get(), UTF_8).split(",");
              if (sizeEntries.length == 2) {
                tabletSize += Long.parseLong(sizeEntries[0]);
              }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
index 1ebbf13,0000000..e71a377
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
@@@ -1,31 -1,0 +1,33 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class MonitorUtil {
 +  public static String getLocation(Instance instance) throws KeeperException, InterruptedException {
 +    ZooReader zr = new ZooReader(instance.getZooKeepers(), 5000);
 +    byte[] loc = zr.getData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, null);
-     return loc==null ? null : new String(loc, Constants.UTF8);
++    return loc==null ? null : new String(loc, UTF_8);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index da5648c,755bee2..b187a76
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -557,14 -437,11 +559,14 @@@ public class Shell extends ShellOption
      } catch (IOException e) {
        log.warn("Unable to load history file at " + historyPath);
      }
 -    
 +
 +    // Turn Ctrl+C into Exception instead of JVM exit
 +    reader.setHandleUserInterrupt(true);
 +
      ShellCompletor userCompletor = null;
 -    
 +
      if (execFile != null) {
-       java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
 -      java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++      java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
        try {
          while (scanner.hasNextLine() && !hasExited()) {
            execCommand(scanner.nextLine(), true, isVerbose());
@@@ -974,14 -805,14 +976,14 @@@
      @Override
      public void close() {}
    };
 -  
 +
    public static class PrintFile implements PrintLine {
      PrintWriter writer;
 -    
 +
      public PrintFile(String filename) throws FileNotFoundException {
-       writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), Constants.UTF8)));
+       writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), UTF_8)));
      }
 -    
 +
      @Override
      public void print(String s) {
        writer.println(s);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
index f06a639,c325612..90a3548
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
@@@ -19,9 -21,7 +21,8 @@@ import static com.google.common.base.Ch
  import java.io.File;
  import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.accumulo.core.util.shell.Shell;
  import org.apache.accumulo.core.util.shell.Shell.Command;
  import org.apache.commons.cli.CommandLine;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
index bcb6c24,5c86ac9..44dce0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
@@@ -35,12 -36,12 +36,12 @@@ public class AuthenticateCommand extend
      final String user = cl.getArgs()[0];
      final String p = shellState.readMaskedLine("Enter current password for '" + user + "': ", '*');
      if (p == null) {
 -      shellState.getReader().printNewline();
 +      shellState.getReader().println();
        return 0;
      } // user canceled
-     final byte[] password = p.getBytes(Constants.UTF8);
+     final byte[] password = p.getBytes(UTF_8);
      final boolean valid = shellState.getConnector().securityOperations().authenticateUser(user, new PasswordToken(password));
 -    shellState.getReader().printString((valid ? "V" : "Not v") + "alid\n");
 +    shellState.getReader().println((valid ? "V" : "Not v") + "alid");
      return 0;
    }
    

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
index d2c73f7,8e39115..fc1af0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
@@@ -16,16 -16,16 +16,17 @@@
   */
  package org.apache.accumulo.core.util.shell.commands;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.io.File;
  import java.io.IOException;
 +import java.util.Map;
  import java.util.Map.Entry;
  import java.util.Scanner;
 +import java.util.Set;
  import java.util.SortedSet;
  import java.util.TreeSet;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.AccumuloException;
  import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.TableExistsException;
@@@ -73,12 -70,12 +74,12 @@@ public class CreateTableCommand extend
      }
      final SortedSet<Text> partitions = new TreeSet<Text>();
      final boolean decode = cl.hasOption(base64Opt.getOpt());
 -    
 +
      if (cl.hasOption(createTableOptSplit.getOpt())) {
        final String f = cl.getOptionValue(createTableOptSplit.getOpt());
 -      
 +
        String line;
-       Scanner file = new Scanner(new File(f), Constants.UTF8.name());
+       Scanner file = new Scanner(new File(f), UTF_8.name());
        try {
          while (file.hasNextLine()) {
            line = file.nextLine();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
index 61f60f8,4f1c793..06d36b9
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
@@@ -19,8 -21,6 +21,7 @@@ import static com.google.common.base.Ch
  import java.security.SecureRandom;
  import java.util.Random;
  
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.accumulo.core.util.shell.Shell;
  import org.apache.accumulo.core.util.shell.Shell.Command;
  import org.apache.accumulo.core.util.shell.ShellCommandException;
@@@ -39,11 -40,12 +40,11 @@@ public class HiddenCommand extends Comm
    public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
      if (rand.nextInt(10) == 0) {
        shellState.getReader().beep();
 -      shellState.getReader().printNewline();
 -      shellState.getReader().printString(
 +      shellState.getReader().println();
 +      shellState.getReader().println(
            new String(Base64.decodeBase64(("ICAgICAgIC4tLS4KICAgICAgLyAvXCBcCiAgICAgKCAvLS1cICkKICAgICAuPl8gIF88LgogICAgLyB8ICd8ICcgXAog"
                + "ICAvICB8Xy58Xy4gIFwKICAvIC98ICAgICAgfFwgXAogfCB8IHwgfFwvfCB8IHwgfAogfF98IHwgfCAgfCB8IHxffAogICAgIC8gIF9fICBcCiAgICAvICAv"
-               + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(Constants.UTF8)), Constants.UTF8));
+               + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(UTF_8)), UTF_8));
 -      shellState.getReader().printNewline();
      } else {
        throw new ShellCommandException(ErrorCode.UNRECOGNIZED_COMMAND, getName());
      }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/PasswdCommand.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
index a0743f4,01fb92e..b8fc4c4
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
@@@ -41,10 -42,10 +42,10 @@@ public class UserCommand extends Comman
      // of these methods fails
      final String p = shellState.readMaskedLine("Enter password for user " + user + ": ", '*');
      if (p == null) {
 -      shellState.getReader().printNewline();
 +      shellState.getReader().println();
        return 0;
      } // user canceled
-     pass = p.getBytes(Constants.UTF8);
+     pass = p.getBytes(UTF_8);
      shellState.updateUser(user, new PasswordToken(pass));
      return 0;
    }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index 259cb06,3794f29..c276056
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@@ -16,6 -16,8 +16,7 @@@
   */
  package org.apache.accumulo.core.client;
  
+ import static com.google.common.base.Charsets.UTF_8;
 -
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertNotEquals;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
index dde5575,0000000..dce742f
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
@@@ -1,157 -1,0 +1,158 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.List;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ZooKeeperInstanceTest {
 +  private static final UUID IID = UUID.randomUUID();
 +  private static final String IID_STRING = IID.toString();
 +  private ClientConfiguration config;
 +  private ZooCacheFactory zcf;
 +  private ZooCache zc;
 +  private ZooKeeperInstance zki;
 +
 +  private void mockIdConstruction(ClientConfiguration config) {
 +    expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
 +    expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
 +    expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
 +    expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
 +  }
 +
 +  private void mockNameConstruction(ClientConfiguration config) {
 +    expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(null);
 +    expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn("instance");
 +    expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
 +    expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
 +  }
 +
 +  @Before
 +  public void setUp() {
 +    config = createMock(ClientConfiguration.class);
 +    mockNameConstruction(config);
 +    replay(config);
 +    zcf = createMock(ZooCacheFactory.class);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache("zk1", 30000)).andReturn(zc).anyTimes();
 +    replay(zcf);
 +    zki = new ZooKeeperInstance(config, zcf);
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void testInvalidConstruction() {
 +    config = createMock(ClientConfiguration.class);
 +    expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
 +    mockNameConstruction(config);
 +    replay(config);
 +    new ZooKeeperInstance(config);
 +  }
 +
 +  @Test
 +  public void testSimpleGetters() {
 +    assertEquals("instance", zki.getInstanceName());
 +    assertEquals("zk1", zki.getZooKeepers());
 +    assertEquals(30000, zki.getZooKeepersSessionTimeOut());
 +  }
 +
 +  @Test
 +  public void testGetInstanceID_FromCache() {
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
 +    replay(zc);
 +    assertEquals(IID_STRING, zki.getInstanceID());
 +  }
 +
 +  @Test
 +  public void testGetInstanceID_Direct() {
 +    config = createMock(ClientConfiguration.class);
 +    mockIdConstruction(config);
 +    replay(config);
 +    zki = new ZooKeeperInstance(config, zcf);
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
 +    replay(zc);
 +    assertEquals(IID_STRING, zki.getInstanceID());
 +  }
 +
 +  @Test(expected = RuntimeException.class)
 +  public void testGetInstanceID_NoMapping() {
 +    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(null);
 +    replay(zc);
 +    zki.getInstanceID();
 +  }
 +
 +  @Test(expected = RuntimeException.class)
 +  public void testGetInstanceID_IDMissingForName() {
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
 +    replay(zc);
 +    zki.getInstanceID();
 +  }
 +
 +  @Test(expected = RuntimeException.class)
 +  public void testGetInstanceID_IDMissingForID() {
 +    config = createMock(ClientConfiguration.class);
 +    mockIdConstruction(config);
 +    replay(config);
 +    zki = new ZooKeeperInstance(config, zcf);
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
 +    replay(zc);
 +    zki.getInstanceID();
 +  }
 +
 +  @Test
 +  public void testGetInstanceName() {
 +    config = createMock(ClientConfiguration.class);
 +    mockIdConstruction(config);
 +    replay(config);
 +    zki = new ZooKeeperInstance(config, zcf);
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add("child1");
 +    children.add("child2");
 +    expect(zc.getChildren(Constants.ZROOT + Constants.ZINSTANCES)).andReturn(children);
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(Constants.UTF8));
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(UTF_8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(UTF_8));
 +    replay(zc);
 +    assertEquals("child2", zki.getInstanceName());
 +  }
 +
 +  @Test
 +  public void testAllZooKeepersAreUsed() {
 +    final String zookeepers = "zk1,zk2,zk3", instanceName = "accumulo";
 +    ZooCacheFactory factory = createMock(ZooCacheFactory.class);
 +    expect(factory.getZooCache(zookeepers, 30000)).andReturn(zc).anyTimes();
 +    replay(factory);
 +    ClientConfiguration cfg = ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers);
 +    ZooKeeperInstance zki = new ZooKeeperInstance(cfg, factory);
 +    assertEquals(zookeepers, zki.getZooKeepers());
 +    assertEquals(instanceName, zki.getInstanceName());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index db4a12b,0000000..e2661e2
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -1,306 -1,0 +1,306 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.accumulo.core.client.mock;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertTrue;
 +import static org.junit.Assert.fail;
 +
 +import java.util.EnumSet;
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.NamespaceNotEmptyException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.NamespaceOperations;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.Filter;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.junit.Test;
 +
 +public class MockNamespacesTest {
 +
 +  Random random = new Random();
 +
 +  /**
 +   * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
 +   */
 +  @Test
 +  public void testDefaultNamespace() throws Exception {
 +    String tableName = "test";
 +    Instance instance = new MockInstance("default");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
 +    c.tableOperations().create(tableName);
 +    assertTrue(c.tableOperations().exists(tableName));
 +  }
 +
 +  /**
 +   * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2"
 +   * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the
 +   * tables and delete the namespace.
 +   */
 +  @Test
 +  public void testCreateAndDeleteNamespace() throws Exception {
 +    String namespace = "testing";
 +    String tableName1 = namespace + ".table1";
 +    String tableName2 = namespace + ".table2";
 +
 +    Instance instance = new MockInstance("createdelete");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.namespaceOperations().create(namespace);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +
 +    c.tableOperations().create(tableName1);
 +    assertTrue(c.tableOperations().exists(tableName1));
 +
 +    c.tableOperations().create(tableName2);
 +    assertTrue(c.tableOperations().exists(tableName2));
 +
 +    // deleting
 +    try {
 +      // can't delete a namespace with tables in it
 +      c.namespaceOperations().delete(namespace);
 +      fail();
 +    } catch (NamespaceNotEmptyException e) {
 +      // ignore, supposed to happen
 +    }
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(tableName1));
 +    assertTrue(c.tableOperations().exists(tableName2));
 +
 +    c.tableOperations().delete(tableName2);
 +    assertTrue(!c.tableOperations().exists(tableName2));
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +
 +    c.tableOperations().delete(tableName1);
 +    assertTrue(!c.tableOperations().exists(tableName1));
 +    c.namespaceOperations().delete(namespace);
 +    assertTrue(!c.namespaceOperations().exists(namespace));
 +  }
 +
 +  /**
 +   * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a
 +   * namespace-wide level, use {@link NamespaceOperations}.
 +   * 
 +   * Checks to make sure namespace-level properties are overridden by table-level properties.
 +   * 
 +   * Checks to see if the default namespace's properties work as well.
 +   */
 +
 +  @Test
 +  public void testNamespaceProperties() throws Exception {
 +    String namespace = "propchange";
 +    String tableName1 = namespace + ".table1";
 +    String tableName2 = namespace + ".table2";
 +
 +    String propKey = Property.TABLE_SCAN_MAXMEM.getKey();
 +    String propVal = "42K";
 +
 +    Instance instance = new MockInstance("props");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName1);
 +    c.namespaceOperations().setProperty(namespace, propKey, propVal);
 +
 +    // check the namespace has the property
 +    assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal));
 +
 +    // check that the table gets it from the namespace
 +    assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
 +
 +    // test a second table to be sure the first wasn't magical
 +    // (also, changed the order, the namespace has the property already)
 +    c.tableOperations().create(tableName2);
 +    assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
 +
 +    // test that table properties override namespace properties
 +    String propKey2 = Property.TABLE_FILE_MAX.getKey();
 +    String propVal2 = "42";
 +    String tablePropVal = "13";
 +
 +    c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
 +    c.namespaceOperations().setProperty("propchange", propKey2, propVal2);
 +
 +    assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
 +
 +    // now check that you can change the default namespace's properties
 +    propVal = "13K";
 +    String tableName = "some_table";
 +    c.tableOperations().create(tableName);
 +    c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal);
 +
 +    assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
 +
 +    // test the properties server-side by configuring an iterator.
 +    // should not show anything with column-family = 'a'
 +    String tableName3 = namespace + ".table3";
 +    c.tableOperations().create(tableName3);
 +
 +    IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName());
 +    c.namespaceOperations().attachIterator(namespace, setting);
 +
 +    BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig());
 +    Mutation m = new Mutation("r");
 +    m.put("a", "b", new Value("abcde".getBytes()));
 +    bw.addMutation(m);
 +    bw.flush();
 +    bw.close();
 +
 +    // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY);
 +    // do scanners work correctly in mock?
 +    // assertTrue(!s.iterator().hasNext());
 +  }
 +
 +  /**
 +   * This test renames and clones two separate table into different namespaces. different namespace.
 +   */
 +  @Test
 +  public void testRenameAndCloneTableToNewNamespace() throws Exception {
 +    String namespace1 = "renamed";
 +    String namespace2 = "cloned";
 +    String tableName = "table";
 +    String tableName1 = "renamed.table1";
 +    // String tableName2 = "cloned.table2";
 +
 +    Instance instance = new MockInstance("renameclone");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.tableOperations().create(tableName);
 +    c.namespaceOperations().create(namespace1);
 +    c.namespaceOperations().create(namespace2);
 +
 +    c.tableOperations().rename(tableName, tableName1);
 +
 +    assertTrue(c.tableOperations().exists(tableName1));
 +    assertTrue(!c.tableOperations().exists(tableName));
 +
 +    // TODO implement clone in mock
 +    /*
 +     * c.tableOperations().clone(tableName1, tableName2, false, null, null);
 +     * 
 +     * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2));
 +     */
 +    return;
 +  }
 +
 +  /**
 +   * This test renames a namespace and ensures that its tables are still correct
 +   */
 +  @Test
 +  public void testNamespaceRename() throws Exception {
 +    String namespace1 = "n1";
 +    String namespace2 = "n2";
 +    String table = "t";
 +
 +    Instance instance = new MockInstance("rename");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.namespaceOperations().create(namespace1);
 +    c.tableOperations().create(namespace1 + "." + table);
 +
 +    c.namespaceOperations().rename(namespace1, namespace2);
 +
 +    assertTrue(!c.namespaceOperations().exists(namespace1));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertTrue(!c.tableOperations().exists(namespace1 + "." + table));
 +    assertTrue(c.tableOperations().exists(namespace2 + "." + table));
 +  }
 +
 +  /**
 +   * This tests adding iterators to a namespace, listing them, and removing them
 +   */
 +  @Test
 +  public void testNamespaceIterators() throws Exception {
 +    Instance instance = new MockInstance("Iterators");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    String namespace = "iterator";
 +    String tableName = namespace + ".table";
 +    String iter = "thing";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName);
 +
 +    IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName());
 +    HashSet<IteratorScope> scope = new HashSet<IteratorScope>();
 +    scope.add(IteratorScope.scan);
 +    c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope));
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m = new Mutation("r");
-     m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
++    m.put("a", "b", new Value("abcde".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +    bw.flush();
 +
 +    Scanner s = c.createScanner(tableName, Authorizations.EMPTY);
 +    System.out.println(s.iterator().next());
 +    // do scanners work correctly in mock?
 +    // assertTrue(!s.iterator().hasNext());
 +
 +    assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter));
 +    c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope));
 +  }
 +
 +  private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
 +    for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
 +      if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
 +        return true;
 +      }
 +    }
 +    return false;
 +  }
 +
 +  private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException,
 +      AccumuloSecurityException {
 +    for (Entry<String,String> e : c.namespaceOperations().getProperties(n)) {
 +      if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
 +        return true;
 +      }
 +    }
 +    return false;
 +  }
 +
 +  public static class SimpleFilter extends Filter {
 +    @Override
 +    public boolean accept(Key k, Value v) {
 +      if (k.getColumnFamily().toString().equals("a"))
 +        return false;
 +      return true;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
index 17d2297,0000000..7bb3fa4
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
@@@ -1,129 -1,0 +1,130 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.security.tokens;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.IOException;
 +import java.net.URL;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
 +import org.junit.Assert;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +public class CredentialProviderTokenTest {
 +
 +  private static boolean isCredentialProviderAvailable = false;
 +
 +  // Keystore contains: {'root.password':'password', 'bob.password':'bob'}
 +  private static String keystorePath;
 +
 +  @BeforeClass
 +  public static void setup() {
 +    try {
 +      Class.forName(CredentialProviderFactoryShim.HADOOP_CRED_PROVIDER_CLASS_NAME);
 +      isCredentialProviderAvailable = true;
 +    } catch (Exception e) {
 +      isCredentialProviderAvailable = false;
 +    }
 +
 +    URL keystoreUrl = CredentialProviderTokenTest.class.getResource("/passwords.jceks");
 +    Assert.assertNotNull(keystoreUrl);
 +    keystorePath = "jceks://file/" + new File(keystoreUrl.getFile()).getAbsolutePath();
 +  }
 +
 +  @Test
 +  public void testPasswordsFromCredentialProvider() throws Exception {
 +    if (!isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
-     Assert.assertArrayEquals("password".getBytes(Constants.UTF8), token.getPassword());
++    Assert.assertArrayEquals("password".getBytes(UTF_8), token.getPassword());
 +
 +    token = new CredentialProviderToken("bob.password", keystorePath);
-     Assert.assertArrayEquals("bob".getBytes(Constants.UTF8), token.getPassword());
++    Assert.assertArrayEquals("bob".getBytes(UTF_8), token.getPassword());
 +  }
 +
 +  @Test
 +  public void testEqualityAfterInit() throws Exception {
 +    if (!isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
 +
 +    CredentialProviderToken uninitializedToken = new CredentialProviderToken();
 +    Properties props = new Properties();
 +    props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
 +    props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
 +    uninitializedToken.init(props);
 +
 +    Assert.assertArrayEquals(token.getPassword(), uninitializedToken.getPassword());
 +  }
 +
 +  @Test
 +  public void testMissingClassesThrowsException() throws Exception {
 +    if (isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    try {
 +      new CredentialProviderToken("root.password", keystorePath);
 +      Assert.fail("Should fail to create CredentialProviderToken when classes are not available");
 +    } catch (IOException e) {
 +      // pass
 +    }
 +  }
 +
 +  @Test
 +  public void cloneReturnsCorrectObject() throws Exception {
 +    if (!isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
 +    CredentialProviderToken clone = token.clone();
 +
 +    Assert.assertEquals(token, clone);
 +    Assert.assertArrayEquals(token.getPassword(), clone.getPassword());
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void missingProperties() throws Exception {
 +    CredentialProviderToken token = new CredentialProviderToken();
 +    token.init(new Properties());
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void missingNameProperty() throws Exception {
 +    CredentialProviderToken token = new CredentialProviderToken();
 +    Properties props = new Properties();
 +    props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
 +    token.init(props);
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void missingProviderProperty() throws Exception {
 +    CredentialProviderToken token = new CredentialProviderToken();
 +    Properties props = new Properties();
 +    props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
 +    token.init(props);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
index 3570610,0000000..d9a47d7
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
@@@ -1,251 -1,0 +1,251 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ConditionTest {
 +  private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
 +  private static final String FAMILY = "family";
 +  private static final String QUALIFIER = "qualifier";
 +  private static final String VISIBILITY = "visibility";
 +  private static final String VALUE = "value";
 +  private static final IteratorSetting[] ITERATORS = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
 +      new IteratorSetting(3, "third", "yetanotherclass")};
 +
 +  private String toString(ByteSequence bs) {
 +    if (bs == null) {
 +      return null;
 +    }
-     return new String(bs.toArray(), Constants.UTF8);
++    return new String(bs.toArray(), UTF_8);
 +  }
 +
 +  private Condition c;
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    c = new Condition(FAMILY, QUALIFIER);
 +  }
 +
 +  @Test
 +  public void testConstruction_CharSequence() {
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteArray() {
-     c = new Condition(FAMILY.getBytes(Constants.UTF8), QUALIFIER.getBytes(Constants.UTF8));
++    c = new Condition(FAMILY.getBytes(UTF_8), QUALIFIER.getBytes(UTF_8));
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testConstruction_Text() {
 +    c = new Condition(new Text(FAMILY), new Text(QUALIFIER));
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteSequence() {
-     c = new Condition(new ArrayByteSequence(FAMILY.getBytes(Constants.UTF8)), new ArrayByteSequence(QUALIFIER.getBytes(Constants.UTF8)));
++    c = new Condition(new ArrayByteSequence(FAMILY.getBytes(UTF_8)), new ArrayByteSequence(QUALIFIER.getBytes(UTF_8)));
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testGetSetTimestamp() {
 +    c.setTimestamp(1234L);
 +    assertEquals(Long.valueOf(1234L), c.getTimestamp());
 +  }
 +
 +  @Test
 +  public void testSetValue_CharSequence() {
 +    c.setValue(VALUE);
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testSetValue_ByteArray() {
-     c.setValue(VALUE.getBytes(Constants.UTF8));
++    c.setValue(VALUE.getBytes(UTF_8));
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testSetValue_Text() {
 +    c.setValue(new Text(VALUE));
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testSetValue_ByteSequence() {
-     c.setValue(new ArrayByteSequence(VALUE.getBytes(Constants.UTF8)));
++    c.setValue(new ArrayByteSequence(VALUE.getBytes(UTF_8)));
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testGetSetVisibility() {
 +    ColumnVisibility vis = new ColumnVisibility(VISIBILITY);
 +    c.setVisibility(vis);
 +    assertEquals(VISIBILITY, toString(c.getVisibility()));
 +  }
 +
 +  @Test
 +  public void testGetSetIterators() {
 +    c.setIterators(ITERATORS);
 +    assertArrayEquals(ITERATORS, c.getIterators());
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void testSetIterators_DuplicateName() {
 +    IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
 +        new IteratorSetting(3, "first", "yetanotherclass")};
 +    c.setIterators(iterators);
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void testSetIterators_DuplicatePriority() {
 +    IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
 +        new IteratorSetting(1, "third", "yetanotherclass")};
 +    c.setIterators(iterators);
 +  }
 +
 +  @Test
 +  public void testEquals() {
 +    ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
 +    c.setVisibility(cvis);
 +    c.setValue(VALUE);
 +    c.setTimestamp(1234L);
 +    c.setIterators(ITERATORS);
 +
 +    // reflexivity
 +    assertTrue(c.equals(c));
 +
 +    // non-nullity
 +    assertFalse(c.equals(null));
 +
 +    // symmetry
 +    Condition c2 = new Condition(FAMILY, QUALIFIER);
 +    c2.setVisibility(cvis);
 +    c2.setValue(VALUE);
 +    c2.setTimestamp(1234L);
 +    c2.setIterators(ITERATORS);
 +    assertTrue(c.equals(c2));
 +    assertTrue(c2.equals(c));
 +
 +    Condition c3 = new Condition("nope", QUALIFIER);
 +    c3.setVisibility(cvis);
 +    c3.setValue(VALUE);
 +    c3.setTimestamp(1234L);
 +    c3.setIterators(ITERATORS);
 +    assertFalse(c.equals(c3));
 +    assertFalse(c3.equals(c));
 +    c3 = new Condition(FAMILY, "nope");
 +    c3.setVisibility(cvis);
 +    c3.setValue(VALUE);
 +    c3.setTimestamp(1234L);
 +    c3.setIterators(ITERATORS);
 +    assertFalse(c.equals(c3));
 +    assertFalse(c3.equals(c));
 +
 +    c2.setVisibility(new ColumnVisibility("sekrit"));
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setVisibility(cvis);
 +    c2.setValue(EMPTY);
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setValue(VALUE);
 +    c2.setTimestamp(2345L);
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setTimestamp(1234L);
 +    c2.setIterators(new IteratorSetting[0]);
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setIterators(ITERATORS);
 +    assertTrue(c.equals(c2));
 +    assertTrue(c2.equals(c));
 +    
 +    // set everything but vis, so its null
 +    Condition c4 = new Condition(FAMILY, QUALIFIER);
 +    c4.setValue(VALUE);
 +    c4.setTimestamp(1234L);
 +    c4.setIterators(ITERATORS);
 +    
 +    assertFalse(c.equals(c4));
 +    assertFalse(c4.equals(c));
 +    
 +    // set everything but timestamp, so its null
 +    Condition c5 = new Condition(FAMILY, QUALIFIER);
 +    c5.setVisibility(cvis);
 +    c5.setValue(VALUE);
 +    c5.setIterators(ITERATORS);
 +    
 +    assertFalse(c.equals(c5));
 +    assertFalse(c5.equals(c));
 +    
 +    // set everything but value
 +    Condition c6 = new Condition(FAMILY, QUALIFIER);
 +    c6.setVisibility(cvis);
 +    c6.setTimestamp(1234L);
 +    c6.setIterators(ITERATORS);
 +    
 +    assertFalse(c.equals(c6));
 +    assertFalse(c6.equals(c));
 +    
 +    // test w/ no optional fields set
 +    Condition c7 = new Condition(FAMILY, QUALIFIER);
 +    Condition c8 = new Condition(FAMILY, QUALIFIER);
 +    assertTrue(c7.equals(c8));
 +    assertTrue(c8.equals(c7));
 +
 +  }
 +
 +  @Test
 +  public void testHashCode() {
 +    ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
 +    c.setVisibility(cvis);
 +    c.setValue(VALUE);
 +    c.setTimestamp(1234L);
 +    c.setIterators(ITERATORS);
 +    int hc1 = c.hashCode();
 +
 +    Condition c2 = new Condition(FAMILY, QUALIFIER);
 +    c2.setVisibility(cvis);
 +    c2.setValue(VALUE);
 +    c2.setTimestamp(1234L);
 +    c2.setIterators(ITERATORS);
 +    assertTrue(c.equals(c2));
 +    assertEquals(hc1, c2.hashCode());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 47f44f1,0000000..908c391
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@@ -1,149 -1,0 +1,149 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ConditionalMutationTest {
-   private static final byte[] ROW = "row".getBytes(Constants.UTF8);
++  private static final byte[] ROW = "row".getBytes(UTF_8);
 +  private static final String FAMILY = "family";
 +  private static final String QUALIFIER = "qualifier";
 +  private static final String QUALIFIER2 = "qualifier2";
 +  private static final String QUALIFIER3 = "qualifier3";
 +
 +  private Condition c1, c2;
 +  private ConditionalMutation cm;
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    c1 = new Condition(FAMILY, QUALIFIER);
 +    c2 = new Condition(FAMILY, QUALIFIER2);
 +    assertFalse(c1.equals(c2));
 +    cm = new ConditionalMutation(ROW, c1, c2);
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteArray() {
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteArray_StartAndLength() {
 +    cm = new ConditionalMutation(ROW, 1, 1, c1, c2);
-     assertArrayEquals("o".getBytes(Constants.UTF8), cm.getRow());
++    assertArrayEquals("o".getBytes(UTF_8), cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_Text() {
 +    cm = new ConditionalMutation(new Text(ROW), c1, c2);
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_CharSequence() {
-     cm = new ConditionalMutation(new String(ROW, Constants.UTF8), c1, c2);
++    cm = new ConditionalMutation(new String(ROW, UTF_8), c1, c2);
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteSequence() {
 +    cm = new ConditionalMutation(new ArrayByteSequence(ROW), c1, c2);
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testCopyConstructor() {
 +    ConditionalMutation cm2 = new ConditionalMutation(cm);
 +    assertArrayEquals(cm.getRow(), cm2.getRow());
 +    assertEquals(cm.getConditions(), cm2.getConditions());
 +  }
 +
 +  @Test
 +  public void testAddCondition() {
 +    Condition c3 = new Condition(FAMILY, QUALIFIER3);
 +    cm.addCondition(c3);
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(3, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +    assertEquals(c3, cs.get(2));
 +  }
 +
 +  @Test
 +  public void testEquals() {
 +    // reflexivity
 +    assertTrue(cm.equals(cm));
 +
 +    // non-nullity
 +    assertFalse(cm.equals((Object) null));
 +
 +    // symmetry
 +    ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
 +    assertTrue(cm.equals(cm2));
 +    assertTrue(cm2.equals(cm));
 +
-     ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(Constants.UTF8), c1, c2);
++    ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(UTF_8), c1, c2);
 +    assertFalse(cm.equals(cm3));
 +    cm3 = new ConditionalMutation(ROW, c2, c1);
 +    assertFalse(cm.getConditions().equals(cm3.getConditions()));
 +    assertFalse(cm.equals(cm3));
 +  }
 +
 +  @Test
 +  public void testEquals_Mutation() {
 +    Mutation m = new Mutation(ROW);
 +    assertFalse(m.equals(cm));
 +    assertFalse(cm.equals(m));
 +  }
 +
 +  @Test
 +  public void testHashcode() {
 +    ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
 +    assertTrue(cm.equals(cm2));
 +    assertEquals(cm2.hashCode(), cm.hashCode());
 +  }
 +}


[43/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
index 5590f7b,0000000..abe5e8c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
@@@ -1,223 -1,0 +1,223 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.replication;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
 +import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
 +
 +import java.io.DataOutputStream;
 +import java.io.File;
- import java.nio.charset.StandardCharsets;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.Map.Entry;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.protobuf.ProtobufUtil;
 +import org.apache.accumulo.core.replication.StatusUtil;
 +import org.apache.accumulo.core.replication.proto.Replication.Status;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.data.ServerMutation;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.logger.LogEvents;
 +import org.apache.accumulo.tserver.logger.LogFileKey;
 +import org.apache.accumulo.tserver.logger.LogFileValue;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Assert;
 +import org.junit.Test;
 +
 +import com.google.common.collect.Iterables;
 +
 +/**
 + *
 + */
 +public class UnusedWalDoesntCloseReplicationStatus extends ConfigurableMacIT {
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
 +    cfg.setNumTservers(1);
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    File accumuloDir = this.getCluster().getConfig().getAccumuloDir();
 +    final Connector conn = getConnector();
 +    final String tableName = getUniqueNames(1)[0];
 +
 +    conn.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
 +    conn.tableOperations().create(tableName);
 +
 +    final String tableId = conn.tableOperations().tableIdMap().get(tableName);
 +    final int numericTableId = Integer.valueOf(tableId);
 +    final int fakeTableId = numericTableId + 1;
 +
 +    Assert.assertNotNull("Did not find table ID", tableId);
 +
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION.getKey(), "true");
 +    conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
 +    // just sleep
 +    conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
 +        ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
 +
 +    FileSystem fs = FileSystem.getLocal(new Configuration());
 +    File tserverWalDir = new File(accumuloDir, ServerConstants.WAL_DIR + Path.SEPARATOR + "faketserver+port");
 +    File tserverWal = new File(tserverWalDir, UUID.randomUUID().toString());
 +    fs.mkdirs(new Path(tserverWalDir.getAbsolutePath()));
 +
 +    // Make a fake WAL with no data in it for our real table
 +    FSDataOutputStream out = fs.create(new Path(tserverWal.getAbsolutePath()));
 +
-     out.write(DfsLogger.LOG_FILE_HEADER_V3.getBytes(StandardCharsets.UTF_8));
++    out.write(DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8));
 +
 +    DataOutputStream dos = new DataOutputStream(out);
 +    dos.writeUTF("NullCryptoModule");
 +
 +    // Fake a single update WAL that has a mutation for another table
 +    LogFileKey key = new LogFileKey();
 +    LogFileValue value = new LogFileValue();
 +
 +    key.event = OPEN;
 +    key.tserverSession = tserverWal.getAbsolutePath();
 +    key.filename = tserverWal.getAbsolutePath();
 +    key.write(out);
 +    value.write(out);
 +
 +    key.event = LogEvents.DEFINE_TABLET;
 +    key.tablet = new KeyExtent(new Text(Integer.toString(fakeTableId)), null, null);
 +    key.seq = 1l;
 +    key.tid = 1;
 +
 +    key.write(dos);
 +    value.write(dos);
 +
 +    key.tablet = null;
 +    key.event = LogEvents.MUTATION;
 +    key.filename = tserverWal.getAbsolutePath();
 +    value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
 +
 +    key.write(dos);
 +    value.write(dos);
 +
 +    key.event = LogEvents.COMPACTION_START;
 +    key.filename = accumuloDir.getAbsolutePath() + "/tables/" + fakeTableId + "/t-000001/A000001.rf";
 +    value.mutations = Collections.emptyList();
 +
 +    key.write(dos);
 +    value.write(dos);
 +
 +    key.event = LogEvents.COMPACTION_FINISH;
 +    value.mutations = Collections.emptyList();
 +
 +    key.write(dos);
 +    value.write(dos);
 +
 +    dos.close();
 +
 +    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m = new Mutation("m");
 +    m.put("m", "m", "M");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    log.info("State of metadata table after inserting a record");
 +
 +    Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
 +    for (Entry<Key,Value> entry : s) {
 +      System.out.println(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
 +    }
 +
 +    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    s.setRange(MetadataSchema.ReplicationSection.getRange());
 +    for (Entry<Key,Value> entry : s) {
 +      System.out.println(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
 +    }
 +
 +    log.info("Offline'ing table");
 +
 +    conn.tableOperations().offline(tableName, true);
 +
 +    // Add our fake WAL to the log column for this table
 +    String walUri = tserverWal.toURI().toString();
 +    KeyExtent extent = new KeyExtent(new Text(tableId), null, null);
 +    bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +    m = new Mutation(extent.getMetadataEntry());
 +    m.put(MetadataSchema.TabletsSection.LogColumnFamily.NAME, new Text("localhost:12345/" + walUri),
-         new Value((walUri + "|1").getBytes(StandardCharsets.UTF_8)));
++        new Value((walUri + "|1").getBytes(UTF_8)));
 +    bw.addMutation(m);
 +
 +    // Add a replication entry for our fake WAL
 +    m = new Mutation(MetadataSchema.ReplicationSection.getRowPrefix() + new Path(walUri).toString());
 +    m.put(MetadataSchema.ReplicationSection.COLF, new Text(tableId), new Value(StatusUtil.fileCreated(System.currentTimeMillis()).toByteArray()));
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    log.info("State of metadata after injecting WAL manually");
 +
 +    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
 +    for (Entry<Key,Value> entry : s) {
 +      log.info(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
 +    }
 +
 +    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    s.setRange(MetadataSchema.ReplicationSection.getRange());
 +    for (Entry<Key,Value> entry : s) {
 +      log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
 +    }
 +
 +    log.info("Bringing table online");
 +    conn.tableOperations().online(tableName, true);
 +
 +    Assert.assertEquals(1, Iterables.size(conn.createScanner(tableName, Authorizations.EMPTY)));
 +
 +    log.info("Table has performed recovery, state of metadata:");
 +
 +    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
 +    for (Entry<Key,Value> entry : s) {
 +      log.info(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
 +    }
 +
 +    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    s.setRange(MetadataSchema.ReplicationSection.getRange());
 +    for (Entry<Key,Value> entry : s) {
 +      Status status = Status.parseFrom(entry.getValue().get());
 +      log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(status));
 +      Assert.assertFalse("Status record was closed and it should not be", status.getClosed());
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
index 0ed92fc,5010801..84e3204
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
@@@ -16,8 -16,8 +16,9 @@@
   */
  package org.apache.accumulo.trace.instrument.receivers;
  
++import static java.nio.charset.StandardCharsets.UTF_8;
++
  import java.io.IOException;
--import java.nio.charset.Charset;
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Map;
@@@ -39,7 -39,7 +40,6 @@@ public class ZooSpanClient extends Send
    private static final Logger log = Logger.getLogger(ZooSpanClient.class);
    private static final int TOTAL_TIME_WAIT_CONNECT_MS = 10 * 1000;
    private static final int TIME_WAIT_CONNECT_CHECK_MS = 100;
-   private static final Charset UTF8 = Charset.forName("UTF-8");
 -  private static final Charset UTF_8 = Charset.forName("UTF-8");
    
    ZooKeeper zoo = null;
    final String path;


[13/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 71b7155,0000000..78f8c7b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -1,318 -1,0 +1,320 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.tables;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.security.SecurityPermission;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.util.TablePropUtil;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.Watcher.Event.EventType;
 +
 +public class TableManager {
 +  private static SecurityPermission TABLE_MANAGER_PERMISSION = new SecurityPermission("tableManagerPermission");
 +
 +  private static final Logger log = Logger.getLogger(TableManager.class);
 +  private static final Set<TableObserver> observers = Collections.synchronizedSet(new HashSet<TableObserver>());
 +  private static final Map<String,TableState> tableStateCache = Collections.synchronizedMap(new HashMap<String,TableState>());
 +  private static final byte[] ZERO_BYTE = new byte[] {'0'};
 +  
 +  private static TableManager tableManager = null;
 +
 +  private final Instance instance;
 +  private ZooCache zooStateCache;
 +
 +  public static void prepareNewNamespaceState(String instanceId, String namespaceId, String namespace, NodeExistsPolicy existsPolicy) throws KeeperException,
 +      InterruptedException {
 +    log.debug("Creating ZooKeeper entries for new namespace " + namespace + " (ID: " + namespaceId + ")");
 +    String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    zoo.putPersistentData(zPath, new byte[0], existsPolicy);
-     zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(Constants.UTF8), existsPolicy);
++    zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(UTF_8), existsPolicy);
 +    zoo.putPersistentData(zPath + Constants.ZNAMESPACE_CONF, new byte[0], existsPolicy);
 +  }
 +
 +  public static void prepareNewTableState(String instanceId, String tableId, String namespaceId, String tableName, TableState state,
 +      NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
 +    // state gets created last
 +    log.debug("Creating ZooKeeper entries for new table " + tableName + " (ID: " + tableId + ") in namespace (ID: " + namespaceId + ")");
 +    Pair<String,String> qualifiedTableName = Tables.qualify(tableName);
 +    tableName = qualifiedTableName.getSecond();
 +    String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId;
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    zoo.putPersistentData(zTablePath, new byte[0], existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_CONF, new byte[0], existsPolicy);
-     zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(Constants.UTF8), existsPolicy);
-     zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(Constants.UTF8), existsPolicy);
++    zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(UTF_8), existsPolicy);
++    zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(UTF_8), existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE, existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE, existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID, ZERO_BYTE, existsPolicy);
-     zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(Constants.UTF8), existsPolicy);
++    zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(UTF_8), existsPolicy);
 +  }
 +
 +  public synchronized static TableManager getInstance() {
 +    SecurityManager sm = System.getSecurityManager();
 +    if (sm != null) {
 +      sm.checkPermission(TABLE_MANAGER_PERMISSION);
 +    }
 +    if (tableManager == null)
 +      tableManager = new TableManager();
 +    return tableManager;
 +  }
 +
 +  private TableManager() {
 +    instance = HdfsZooInstance.getInstance();
 +    zooStateCache = new ZooCache(new TableStateWatcher());
 +    updateTableStateCache();
 +  }
 +
 +  public TableState getTableState(String tableId) {
 +    return tableStateCache.get(tableId);
 +  }
 +
 +  public static class IllegalTableTransitionException extends Exception {
 +    private static final long serialVersionUID = 1L;
 +
 +    final TableState oldState;
 +    final TableState newState;
 +
 +    public IllegalTableTransitionException(TableState oldState, TableState newState) {
 +      this.oldState = oldState;
 +      this.newState = newState;
 +    }
 +
 +    public TableState getOldState() {
 +      return oldState;
 +    }
 +
 +    public TableState getNewState() {
 +      return newState;
 +    }
 +
 +  }
 +
 +  public synchronized void transitionTableState(final String tableId, final TableState newState) {
 +    String statePath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE;
 +
 +    try {
-       ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(Constants.UTF8), ZooUtil.PUBLIC, new Mutator() {
++      ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(UTF_8), ZooUtil.PUBLIC, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] oldData) throws Exception {
 +          TableState oldState = TableState.UNKNOWN;
 +          if (oldData != null)
-             oldState = TableState.valueOf(new String(oldData, Constants.UTF8));
++            oldState = TableState.valueOf(new String(oldData, UTF_8));
 +          boolean transition = true;
 +          // +--------+
 +          // v |
 +          // NEW -> (ONLINE|OFFLINE)+--- DELETING
 +          switch (oldState) {
 +            case NEW:
 +              transition = (newState == TableState.OFFLINE || newState == TableState.ONLINE);
 +              break;
 +            case ONLINE: // fall-through intended
 +            case UNKNOWN:// fall through intended
 +            case OFFLINE:
 +              transition = (newState != TableState.NEW);
 +              break;
 +            case DELETING:
 +              // Can't transition to any state from DELETING
 +              transition = false;
 +              break;
 +          }
 +          if (!transition)
 +            throw new IllegalTableTransitionException(oldState, newState);
 +          log.debug("Transitioning state for table " + tableId + " from " + oldState + " to " + newState);
-           return newState.name().getBytes(Constants.UTF8);
++          return newState.name().getBytes(UTF_8);
 +        }
 +      });
 +    } catch (Exception e) {
 +      log.fatal("Failed to transition table to state " + newState);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  private void updateTableStateCache() {
 +    synchronized (tableStateCache) {
 +      for (String tableId : zooStateCache.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES))
 +        if (zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE) != null)
 +          updateTableStateCache(tableId);
 +    }
 +  }
 +
 +  public TableState updateTableStateCache(String tableId) {
 +    synchronized (tableStateCache) {
 +      TableState tState = TableState.UNKNOWN;
 +      byte[] data = zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE);
 +      if (data != null) {
-         String sState = new String(data, Constants.UTF8);
++        String sState = new String(data, UTF_8);
 +        try {
 +          tState = TableState.valueOf(sState);
 +        } catch (IllegalArgumentException e) {
 +          log.error("Unrecognized state for table with tableId=" + tableId + ": " + sState);
 +        }
 +        tableStateCache.put(tableId, tState);
 +      }
 +      return tState;
 +    }
 +  }
 +
 +  public void addTable(String tableId, String namespaceId, String tableName, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException,
 +      NamespaceNotFoundException {
 +    prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
 +    updateTableStateCache(tableId);
 +  }
 +
 +  public void cloneTable(String srcTable, String tableId, String tableName, String namespaceId, Map<String,String> propertiesToSet,
 +      Set<String> propertiesToExclude, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
 +    prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
 +
 +    String srcTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + srcTable + Constants.ZTABLE_CONF;
 +    String newTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF;
 +    ZooReaderWriter.getInstance().recursiveCopyPersistent(srcTablePath, newTablePath, NodeExistsPolicy.OVERWRITE);
 +
 +    for (Entry<String,String> entry : propertiesToSet.entrySet())
 +      TablePropUtil.setTableProperty(tableId, entry.getKey(), entry.getValue());
 +
 +    for (String prop : propertiesToExclude)
 +      ZooReaderWriter.getInstance().recursiveDelete(
 +          Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF + "/" + prop, NodeMissingPolicy.SKIP);
 +
 +    updateTableStateCache(tableId);
 +  }
 +
 +  public void removeTable(String tableId) throws KeeperException, InterruptedException {
 +    synchronized (tableStateCache) {
 +      tableStateCache.remove(tableId);
 +      ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE,
 +          NodeMissingPolicy.SKIP);
 +      ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId, NodeMissingPolicy.SKIP);
 +    }
 +  }
 +
 +  public boolean addObserver(TableObserver to) {
 +    synchronized (observers) {
 +      synchronized (tableStateCache) {
 +        to.initialize(Collections.unmodifiableMap(tableStateCache));
 +        return observers.add(to);
 +      }
 +    }
 +  }
 +
 +  public boolean removeObserver(TableObserver to) {
 +    return observers.remove(to);
 +  }
 +
 +  private class TableStateWatcher implements Watcher {
 +    @Override
 +    public void process(WatchedEvent event) {
 +      if (log.isTraceEnabled())
 +        log.trace(event);
 +
 +      final String zPath = event.getPath();
 +      final EventType zType = event.getType();
 +
 +      String tablesPrefix = ZooUtil.getRoot(instance) + Constants.ZTABLES;
 +      String tableId = null;
 +
 +      if (zPath != null && zPath.startsWith(tablesPrefix + "/")) {
 +        String suffix = zPath.substring(tablesPrefix.length() + 1);
 +        if (suffix.contains("/")) {
 +          String[] sa = suffix.split("/", 2);
 +          if (Constants.ZTABLE_STATE.equals("/" + sa[1]))
 +            tableId = sa[0];
 +        }
 +        if (tableId == null) {
 +          log.warn("Unknown path in " + event);
 +          return;
 +        }
 +      }
 +
 +      switch (zType) {
 +        case NodeChildrenChanged:
 +          if (zPath != null && zPath.equals(tablesPrefix)) {
 +            updateTableStateCache();
 +          } else {
 +            log.warn("Unexpected path " + zPath);
 +          }
 +          break;
 +        case NodeCreated:
 +        case NodeDataChanged:
 +          // state transition
 +          TableState tState = updateTableStateCache(tableId);
 +          log.debug("State transition to " + tState + " @ " + event);
 +          synchronized (observers) {
 +            for (TableObserver to : observers)
 +              to.stateChanged(tableId, tState);
 +          }
 +          break;
 +        case NodeDeleted:
 +          if (zPath != null
 +              && tableId != null
 +              && (zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_STATE) || zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_CONF) || zPath
 +                  .equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_NAME)))
 +            tableStateCache.remove(tableId);
 +          break;
 +        case None:
 +          switch (event.getState()) {
 +            case Expired:
 +              if (log.isTraceEnabled())
 +                log.trace("Session expired " + event);
 +              synchronized (observers) {
 +                for (TableObserver to : observers)
 +                  to.sessionExpired();
 +              }
 +              break;
 +            case SyncConnected:
 +            default:
 +              if (log.isTraceEnabled())
 +                log.trace("Ignored " + event);
 +          }
 +          break;
 +        default:
 +          log.warn("Unandled " + event);
 +      }
 +    }
 +  }
 +
 +  public void removeNamespace(String namespaceId) throws KeeperException, InterruptedException {
 +    ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId, NodeMissingPolicy.SKIP);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index eabdc0d,0000000..e10c0c1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -1,79 -1,0 +1,81 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.tablets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Random;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.FastFormat;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +/**
 + * Allocates unique names for an accumulo instance. The names are unique for the lifetime of the instance.
 + * 
 + * This is useful for filenames because it makes caching easy.
 + * 
 + */
 +
 +public class UniqueNameAllocator {
 +  private long next = 0;
 +  private long maxAllocated = 0;
 +  private String nextNamePath;
 +  private Random rand;
 +  
 +  private UniqueNameAllocator() {
 +    nextNamePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZNEXT_FILE;
 +    rand = new Random();
 +  }
 +  
 +  public synchronized String getNextName() {
 +    
 +    while (next >= maxAllocated) {
 +      final int allocate = 100 + rand.nextInt(100);
 +      
 +      try {
 +        byte[] max = ZooReaderWriter.getInstance().mutate(nextNamePath, null, ZooUtil.PRIVATE, new ZooReaderWriter.Mutator() {
 +          public byte[] mutate(byte[] currentValue) throws Exception {
-             long l = Long.parseLong(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++            long l = Long.parseLong(new String(currentValue, UTF_8), Character.MAX_RADIX);
 +            l += allocate;
-             return Long.toString(l, Character.MAX_RADIX).getBytes(Constants.UTF8);
++            return Long.toString(l, Character.MAX_RADIX).getBytes(UTF_8);
 +          }
 +        });
 +        
-         maxAllocated = Long.parseLong(new String(max, Constants.UTF8), Character.MAX_RADIX);
++        maxAllocated = Long.parseLong(new String(max, UTF_8), Character.MAX_RADIX);
 +        next = maxAllocated - allocate;
 +        
 +      } catch (Exception e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +    
-     return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), Constants.UTF8);
++    return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), UTF_8);
 +  }
 +  
 +  private static UniqueNameAllocator instance = null;
 +  
 +  public static synchronized UniqueNameAllocator getInstance() {
 +    if (instance == null)
 +      instance = new UniqueNameAllocator();
 +    
 +    return instance;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index cdff21d,0000000..198b2d6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,137 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.BatchWriterOpts;
 +import org.apache.accumulo.core.client.MultiTableBatchWriter;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.cli.ClientOpts;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class AddFilesWithMissingEntries {
 +  
 +  static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
 +  
 +  public static class Opts extends ClientOpts {
 +    @Parameter(names = "-update", description = "Make changes to the " + MetadataTable.NAME + " table to include missing files")
 +    boolean update = false;
 +  }
 +  
 +  /**
 +   * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who knows what
 +   * they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
 +   */
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    BatchWriterOpts bwOpts = new BatchWriterOpts();
 +    opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
 +    
 +    final Scanner scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    scanner.setRange(MetadataSchema.TabletsSection.getRange());
 +    final Configuration conf = new Configuration();
 +    final FileSystem fs = FileSystem.get(conf);
 +    
 +    KeyExtent last = new KeyExtent();
 +    String directory = null;
 +    Set<String> knownFiles = new HashSet<String>();
 +    
 +    int count = 0;
 +    final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
 +    
 +    // collect the list of known files and the directory for each extent
 +    for (Entry<Key,Value> entry : scanner) {
 +      Key key = entry.getKey();
 +      KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
 +      // when the key extent changes
 +      if (!ke.equals(last)) {
 +        if (directory != null) {
 +          // add any files in the directory unknown to the key extent
 +          count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
 +        }
 +        directory = null;
 +        knownFiles.clear();
 +        last = ke;
 +      }
 +      if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
 +        directory = entry.getValue().toString();
 +        log.debug("Found directory " + directory + " for row " + key.getRow().toString());
 +      } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
 +        String filename = key.getColumnQualifier().toString();
 +        knownFiles.add(filename);
 +        log.debug("METADATA file found: " + filename);
 +      }
 +    }
 +    if (directory != null) {
 +      // catch the last key extent
 +      count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
 +    }
 +    log.info("There were " + count + " files that are unknown to the metadata table");
 +    writer.close();
 +  }
 +  
 +  private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update)
 +      throws Exception {
 +    int count = 0;
 +    final String tableId = ke.getTableId().toString();
 +    final Text row = ke.getMetadataEntry();
 +    log.info(row.toString());
 +    for (String dir : ServerConstants.getTablesDirs()) {
 +      final Path path = new Path(dir + "/" + tableId + directory);
 +      for (FileStatus file : fs.listStatus(path)) {
 +        if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
 +          continue;
 +        final String filename = directory + "/" + file.getPath().getName();
 +        if (!knownFiles.contains(filename)) {
 +          count++;
 +          final Mutation m = new Mutation(row);
 +          String size = Long.toString(file.getLen());
 +          String entries = "1"; // lie
 +          String value = size + "," + entries;
-           m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
++          m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(UTF_8)));
 +          if (update) {
 +            writer.getBatchWriter(MetadataTable.NAME).addMutation(m);
 +          }
 +        }
 +      }
 +    }
 +    return count;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 2926a3f,0000000..7a1e1ba
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -1,166 -1,0 +1,167 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.List;
 +import java.util.UUID;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.volume.Volume;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.cli.ClientOpts;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.zookeeper.ZooDefs.Ids;
 +import org.apache.zookeeper.data.ACL;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class ChangeSecret {
 +  
 +  static class Opts extends ClientOpts {
 +    @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
 +    String oldPass;
 +    @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
 +    String newPass;
 +  }
 +  
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    List<String> argsList = new ArrayList<String>(args.length + 2);
 +    argsList.add("--old");
 +    argsList.add("--new");
 +    argsList.addAll(Arrays.asList(args));
 +    opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Instance inst = opts.getInstance();
 +    if (!verifyAccumuloIsDown(inst, opts.oldPass))
 +      System.exit(-1);
 +    String instanceId = rewriteZooKeeperInstance(inst, opts.oldPass, opts.newPass);
 +    updateHdfs(fs, inst, instanceId);
 +    if (opts.oldPass != null) {
 +      deleteInstance(inst, opts.oldPass);
 +    }
 +    System.out.println("New instance id is " + instanceId);
 +    System.out.println("Be sure to put your new secret in accumulo-site.xml");
 +  }
 +  
 +  interface Visitor {
 +    void visit(ZooReader zoo, String path) throws Exception;
 +  }
 +  
 +  private static void recurse(ZooReader zoo, String root, Visitor v) {
 +    try {
 +      v.visit(zoo, root);
 +      for (String child : zoo.getChildren(root)) {
 +        recurse(zoo, root + "/" + child, v);
 +      }
 +    } catch (Exception ex) {
 +      throw new RuntimeException(ex);
 +    }
 +  }
 +  
 +  private static boolean verifyAccumuloIsDown(Instance inst, String oldPassword) {
 +    ZooReader zooReader = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPassword);
 +    String root = ZooUtil.getRoot(inst);
 +    final List<String> ephemerals = new ArrayList<String>();
 +    recurse(zooReader, root, new Visitor() {
 +      public void visit(ZooReader zoo, String path) throws Exception {
 +        Stat stat = zoo.getStatus(path);
 +        if (stat.getEphemeralOwner() != 0)
 +          ephemerals.add(path);
 +      }
 +    });
 +    if (ephemerals.size() == 0) {
 +      return true;
 +    }
 +    
 +    System.err.println("The following ephemeral nodes exist, something is still running:");
 +    for (String path : ephemerals) {
 +      System.err.println(path);
 +    }
 +    return false;
 +  }
 +  
 +  private static String rewriteZooKeeperInstance(final Instance inst, String oldPass, String newPass) throws Exception {
 +    final ZooReaderWriter orig = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPass);
 +    final IZooReaderWriter new_ = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), newPass);
 +    final String newInstanceId = UUID.randomUUID().toString();
 +    String root = ZooUtil.getRoot(inst);
 +    recurse(orig, root, new Visitor() {
 +      public void visit(ZooReader zoo, String path) throws Exception {
 +        String newPath = path.replace(inst.getInstanceID(), newInstanceId);
 +        byte[] data = zoo.getData(path, null);
 +        List<ACL> acls = orig.getZooKeeper().getACL(path, new Stat());
 +        if (acls.containsAll(Ids.READ_ACL_UNSAFE)) {
 +          new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +        } else {
 +          // upgrade
 +          if (acls.containsAll(Ids.OPEN_ACL_UNSAFE)) {
 +            // make user nodes private, they contain the user's password
 +            String parts[] = path.split("/");
 +            if (parts[parts.length - 2].equals("users")) {
 +              new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +            } else {
 +              // everything else can have the readable acl
 +              new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +            }
 +          } else {
 +            new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +          }
 +        }
 +      }
 +    });
 +    String path = "/accumulo/instances/" + inst.getInstanceName();
 +    orig.recursiveDelete(path, NodeMissingPolicy.SKIP);
-     new_.putPersistentData(path, newInstanceId.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    new_.putPersistentData(path, newInstanceId.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    return newInstanceId;
 +  }
 +  
 +  private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
 +    // Need to recreate the instanceId on all of them to keep consistency
 +    for (Volume v : fs.getVolumes()) {
 +      final Path instanceId = ServerConstants.getInstanceIdLocation(v);
 +      if (!v.getFileSystem().delete(instanceId, true)) {
 +        throw new IOException("Could not recursively delete " + instanceId);
 +      }
 +
 +      if (!v.getFileSystem().mkdirs(instanceId)) {
 +        throw new IOException("Could not create directory " + instanceId);
 +      }
 +
 +      v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
 +    }
 +  }
 +  
 +  private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
 +    IZooReaderWriter orig = new ZooReaderWriter(origInstance.getZooKeepers(), origInstance.getZooKeepersSessionTimeOut(), oldPass);
 +    orig.recursiveDelete("/accumulo/" + origInstance.getInstanceID(), NodeMissingPolicy.SKIP);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index f2074a1,0000000..809a7c8
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -1,87 -1,0 +1,89 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class CleanZookeeper {
 +  
 +  private static final Logger log = Logger.getLogger(CleanZookeeper.class);
 +  
 +  static class Opts extends Help {
 +    @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
 +    String keepers = "localhost:2181";
 +    @Parameter(names={"--password"}, description="the system secret", password=true)
 +    String auth;
 +  }
 +  
 +  /**
 +   * @param args
 +   *          must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
 +   * @throws IOException
 +   *           error connecting to accumulo or zookeeper
 +   */
 +  public static void main(String[] args) throws IOException {
 +    Opts opts = new Opts();
 +    opts.parseArgs(CleanZookeeper.class.getName(), args);
 +    
 +    String root = Constants.ZROOT;
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +    if (opts.auth != null) {
-       zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(Constants.UTF8));
++      zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(UTF_8));
 +    }
 +    
 +    try {
 +      for (String child : zk.getChildren(root)) {
 +        if (Constants.ZINSTANCES.equals("/" + child)) {
 +          for (String instanceName : zk.getChildren(root + Constants.ZINSTANCES)) {
 +            String instanceNamePath = root + Constants.ZINSTANCES + "/" + instanceName;
 +            byte[] id = zk.getData(instanceNamePath, null);
-             if (id != null && !new String(id, Constants.UTF8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
++            if (id != null && !new String(id, UTF_8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
 +              try {
 +                zk.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
 +              } catch (KeeperException.NoAuthException ex) {
 +                log.warn("Unable to delete " + instanceNamePath);
 +              }
 +            }
 +          }
 +        } else if (!child.equals(HdfsZooInstance.getInstance().getInstanceID())) {
 +          String path = root + "/" + child;
 +          try {
 +            zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
 +          } catch (KeeperException.NoAuthException ex) {
 +            log.warn("Unable to delete " + path);
 +          }
 +        }
 +      }
 +    } catch (Exception ex) {
 +      System.out.println("Error Occurred: " + ex);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 448da86,0000000..d40830f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -1,81 -1,0 +1,83 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class DeleteZooInstance {
 +  
 +  static class Opts extends Help {
 +    @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
 +    String instance;
 +  }
 +  
 +  static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
 +    for (int i = 0; i < 10; i++){
 +      try {
 +        zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
 +        return;
 +      } catch (KeeperException.NotEmptyException ex) {
 +        // ignored
 +      } catch (Exception ex) {
 +        throw ex;
 +      }
 +    }
 +  }
 +  
 +  /**
 +   * @param args
 +   *          : the name or UUID of the instance to be deleted
 +   */
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    opts.parseArgs(DeleteZooInstance.class.getName(), args);
 +    
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +    // try instance name:
 +    Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
 +    Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
 +    uuids.remove("instances");
 +    if (instances.contains(opts.instance)) {
 +      String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
 +      byte[] data = zk.getData(path, null);
 +      deleteRetry(zk, path);
-       deleteRetry(zk, Constants.ZROOT + "/" + new String(data, Constants.UTF8));
++      deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
 +    } else if (uuids.contains(opts.instance)) {
 +      // look for the real instance name
 +      for (String instance : instances) {
 +        String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
 +        byte[] data = zk.getData(path, null);
-         if (opts.instance.equals(new String(data, Constants.UTF8)))
++        if (opts.instance.equals(new String(data, UTF_8)))
 +          deleteRetry(zk, path);
 +      }
 +      deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 504956f,0000000..43348df
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,121 -1,0 +1,122 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.PrintStream;
 +import java.io.UnsupportedEncodingException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class DumpZookeeper {
 +  
 +  static IZooReaderWriter zk = null;
 +  
 +  private static final Logger log = Logger.getLogger(DumpZookeeper.class);
 +  
 +  private static class Encoded {
 +    public String encoding;
 +    public String value;
 +    
 +    Encoded(String e, String v) {
 +      encoding = e;
 +      value = v;
 +    }
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = "--root", description = "the root of the znode tree to dump")
 +    String root = "/";
 +  }
 +  
 +  public static void main(String[] args) {
 +    Opts opts = new Opts();
 +    opts.parseArgs(DumpZookeeper.class.getName(), args);
 +    
 +    Logger.getRootLogger().setLevel(Level.WARN);
 +    PrintStream out = System.out;
 +    try {
 +      zk = ZooReaderWriter.getInstance();
 +      
 +      write(out, 0, "<dump root='%s'>", opts.root);
 +      for (String child : zk.getChildren(opts.root, null))
 +        if (!child.equals("zookeeper"))
 +          dump(out, opts.root, child, 1);
 +      write(out, 0, "</dump>");
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +  
 +  private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
 +    String path = root + "/" + child;
 +    if (root.endsWith("/"))
 +      path = root + child;
 +    Stat stat = zk.getStatus(path);
 +    if (stat == null)
 +      return;
 +    String type = "node";
 +    if (stat.getEphemeralOwner() != 0) {
 +      type = "ephemeral";
 +    }
 +    if (stat.getNumChildren() == 0) {
 +      if (stat.getDataLength() == 0) {
 +        write(out, indent, "<%s name='%s'/>", type, child);
 +      } else {
 +        Encoded value = value(path);
 +        write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
 +      }
 +    } else {
 +      if (stat.getDataLength() == 0) {
 +        write(out, indent, "<%s name='%s'>", type, child);
 +      } else {
 +        Encoded value = value(path);
 +        write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
 +      }
 +      for (String c : zk.getChildren(path, null)) {
 +        dump(out, path, c, indent + 1);
 +      }
 +      write(out, indent, "</node>");
 +    }
 +  }
 +  
 +  private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
 +    byte[] data = zk.getData(path, null);
 +    for (int i = 0; i < data.length; i++) {
 +      // does this look like simple ascii?
 +      if (data[i] < ' ' || data[i] > '~')
 +        return new Encoded("base64", Base64.encodeBase64String(data));
 +    }
-     return new Encoded(Constants.UTF8.name(), new String(data, Constants.UTF8));
++    return new Encoded(UTF_8.name(), new String(data, UTF_8));
 +  }
 +  
 +  private static void write(PrintStream out, int indent, String fmt, Object... args) {
 +    for (int i = 0; i < indent; i++)
 +      out.print(" ");
 +    out.println(String.format(fmt, args));
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f4655dc,0000000..be85b1b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -1,225 -1,0 +1,227 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Formattable;
 +import java.util.Formatter;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class ListInstances {
 +  
 +  private static final Logger log = Logger.getLogger(ListInstances.class);
 +  
 +  private static final int NAME_WIDTH = 20;
 +  private static final int UUID_WIDTH = 37;
 +  private static final int MASTER_WIDTH = 30;
 +  
 +  private static final int ZOOKEEPER_TIMER_MILLIS = 30 * 1000;
 +  
 +  static class Opts extends Help {
 +    @Parameter(names="--print-errors", description="display errors while listing instances")
 +    boolean printErrors = false;
 +    @Parameter(names="--print-all", description="print information for all instances, not just those with names")
 +    boolean printAll = false;
 +    @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact")
 +    String keepers = null;
 +  }
 +  static Opts opts = new Opts();
 +  static int errors = 0;
 +  
 +  public static void main(String[] args) {
 +    opts.parseArgs(ListInstances.class.getName(), args);
 +    
 +    if (opts.keepers == null) {
 +      opts.keepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
 +    }
 +    
 +    String keepers = opts.keepers;
 +    boolean printAll = opts.printAll;
 +    boolean printErrors = opts.printErrors;
 +    
 +    listInstances(keepers, printAll, printErrors);
 +    
 +  }
 +
 +  static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) {
 +    errors = 0;
 +
 +    System.out.println("INFO : Using ZooKeepers " + keepers);
 +    ZooReader rdr = new ZooReader(keepers, ZOOKEEPER_TIMER_MILLIS);
 +    ZooCache cache = new ZooCache(keepers, ZOOKEEPER_TIMER_MILLIS);
 +
 +    TreeMap<String,UUID> instanceNames = getInstanceNames(rdr, printErrors);
 +    
 +    System.out.println();
 +    printHeader();
 +    
 +    for (Entry<String,UUID> entry : instanceNames.entrySet()) {
 +      printInstanceInfo(cache, entry.getKey(), entry.getValue(), printErrors);
 +    }
 +    
 +    TreeSet<UUID> instancedIds = getInstanceIDs(rdr, printErrors);
 +    instancedIds.removeAll(instanceNames.values());
 +    
 +    if (printAll) {
 +      for (UUID uuid : instancedIds) {
 +        printInstanceInfo(cache, null, uuid, printErrors);
 +      }
 +    } else if (instancedIds.size() > 0) {
 +      System.out.println();
 +      System.out.println("INFO : " + instancedIds.size() + " unamed instances were not printed, run with --print-all to see all instances");
 +    } else {
 +      System.out.println();
 +    }
 +    
 +    if (!printErrors && errors > 0) {
 +      System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info");
 +    }
 +  }
 +  
 +  private static class CharFiller implements Formattable {
 +    
 +    char c;
 +    
 +    CharFiller(char c) {
 +      this.c = c;
 +    }
 +    
 +    @Override
 +    public void formatTo(Formatter formatter, int flags, int width, int precision) {
 +      
 +      StringBuilder sb = new StringBuilder();
 +      for (int i = 0; i < width; i++)
 +        sb.append(c);
 +      formatter.format(sb.toString());
 +    }
 +    
 +  }
 +  
 +  private static void printHeader() {
 +    System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MASTER_WIDTH + "s%n", "Instance Name", "Instance ID", "Master");
 +    System.out.printf("%" + (NAME_WIDTH + 1) + "s+%" + (UUID_WIDTH + 1) + "s+%" + (MASTER_WIDTH + 1) + "s%n", new CharFiller('-'), new CharFiller('-'),
 +        new CharFiller('-'));
 +    
 +  }
 +  
 +  private static void printInstanceInfo(ZooCache cache, String instanceName, UUID iid, boolean printErrors) {
 +    String master = getMaster(cache, iid, printErrors);
 +    if (instanceName == null) {
 +      instanceName = "";
 +    }
 +    
 +    if (master == null) {
 +      master = "";
 +    }
 +    
 +    System.out.printf("%" + NAME_WIDTH + "s |%" + UUID_WIDTH + "s |%" + MASTER_WIDTH + "s%n", "\"" + instanceName + "\"", iid, master);
 +  }
 +  
 +  private static String getMaster(ZooCache cache, UUID iid, boolean printErrors) {
 +    
 +    if (iid == null) {
 +      return null;
 +    }
 +    
 +    try {
 +      String masterLocPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK;
 +      byte[] master = ZooLock.getLockData(cache, masterLocPath, null);
 +      if (master == null) {
 +        return null;
 +      }
-       return new String(master, Constants.UTF8);
++      return new String(master, UTF_8);
 +    } catch (Exception e) {
 +      handleException(e, printErrors);
 +      return null;
 +    }
 +  }
 +  
 +  private static TreeMap<String,UUID> getInstanceNames(ZooReader zk, boolean printErrors) {
 +    
 +    String instancesPath = Constants.ZROOT + Constants.ZINSTANCES;
 +    
 +    TreeMap<String,UUID> tm = new TreeMap<String,UUID>();
 +    
 +    List<String> names;
 +    
 +    try {
 +      names = zk.getChildren(instancesPath);
 +    } catch (Exception e) {
 +      handleException(e, printErrors);
 +      return tm;
 +    }
 +    
 +    for (String name : names) {
 +      String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
 +      try {
-         UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), Constants.UTF8));
++        UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), UTF_8));
 +        tm.put(name, iid);
 +      } catch (Exception e) {
 +        handleException(e, printErrors);
 +        tm.put(name, null);
 +      }
 +    }
 +    
 +    return tm;
 +  }
 +  
 +  private static TreeSet<UUID> getInstanceIDs(ZooReader zk, boolean printErrors) {
 +    TreeSet<UUID> ts = new TreeSet<UUID>();
 +    
 +    try {
 +      List<String> children = zk.getChildren(Constants.ZROOT);
 +      
 +      for (String iid : children) {
 +        if (iid.equals("instances"))
 +          continue;
 +        try {
 +          ts.add(UUID.fromString(iid));
 +        } catch (Exception e) {
 +          log.error("Exception: " + e);
 +        }
 +      }
 +    } catch (Exception e) {
 +      handleException(e, printErrors);
 +    }
 +    
 +    return ts;
 +  }
 +  
 +  private static void handleException(Exception e, boolean printErrors) {
 +    if (printErrors) {
 +      e.printStackTrace();
 +    }
 +    
 +    errors++;
 +  }
 +}


[45/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
----------------------------------------------------------------------
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
index e37d78a,0000000..d389c0b
mode 100644,000000..100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
@@@ -1,100 -1,0 +1,101 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.replication;
 +
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.util.HashMap;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.replication.ReplicaSystem;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.replication.ReplicaSystemHelper;
 +import org.apache.accumulo.core.replication.ReplicationTarget;
 +import org.apache.accumulo.core.replication.proto.Replication.Status;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.replication.DistributedWorkQueueWorkAssignerHelper;
 +import org.apache.hadoop.fs.Path;
 +import org.easymock.EasyMock;
 +import org.junit.Assert;
 +import org.junit.Test;
 +
 +/**
 + * 
 + */
 +public class ReplicationProcessorTest {
 +
 +  @Test
 +  public void peerTypeExtractionFromConfiguration() {
 +    Instance inst = EasyMock.createMock(Instance.class);
 +    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
 +    Credentials creds = new Credentials("foo", new PasswordToken("bar"));
 +
 +    Map<String,String> data = new HashMap<>();
 +
 +    String peerName = "peer";
 +    String configuration = "java.lang.String,foo";
 +    data.put(Property.REPLICATION_PEERS + peerName, configuration);
 +    ConfigurationCopy conf = new ConfigurationCopy(data);
 +
 +    ReplicationProcessor proc = new ReplicationProcessor(inst, conf, fs, creds);
 +
 +    Assert.assertEquals(configuration, proc.getPeerType(peerName));
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void noPeerConfigurationThrowsAnException() {
 +    Instance inst = EasyMock.createMock(Instance.class);
 +    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
 +    Credentials creds = new Credentials("foo", new PasswordToken("bar"));
 +
 +    Map<String,String> data = new HashMap<>();
 +    ConfigurationCopy conf = new ConfigurationCopy(data);
 +
 +    ReplicationProcessor proc = new ReplicationProcessor(inst, conf, fs, creds);
 +
 +    proc.getPeerType("foo");
 +  }
 +
 +  @Test
 +  public void filesWhichMakeNoProgressArentReplicatedAgain() throws Exception {
 +    ReplicaSystem replica = EasyMock.createMock(ReplicaSystem.class);
 +    ReplicaSystemHelper helper = EasyMock.createMock(ReplicaSystemHelper.class);
 +    ReplicationProcessor proc = EasyMock.createMockBuilder(ReplicationProcessor.class).addMockedMethods("getReplicaSystem", "doesFileExist", "getStatus", "getHelper").createMock();
 +
 +    ReplicationTarget target = new ReplicationTarget("peer", "1", "1");
 +    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
 +    Path path = new Path("/accumulo");
 +
 +    String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.toString(), target);
 +
 +    EasyMock.expect(proc.getReplicaSystem(target)).andReturn(replica);
 +    EasyMock.expect(proc.getStatus(path.toString(), target)).andReturn(status);
 +    EasyMock.expect(proc.doesFileExist(path, target)).andReturn(true);
 +    EasyMock.expect(proc.getHelper()).andReturn(helper);
 +    EasyMock.expect(replica.replicate(path, status, target, helper)).andReturn(status);
 +
 +    EasyMock.replay(replica, proc);
 +
-     proc.process(queueKey, path.toString().getBytes(StandardCharsets.UTF_8));
++    proc.process(queueKey, path.toString().getBytes(UTF_8));
 +
 +    EasyMock.verify(replica, proc);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/Shell.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/Shell.java
index 2ebdb6c,0000000..a0ff17a
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@@ -1,1229 -1,0 +1,1231 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++import static java.nio.charset.StandardCharsets.ISO_8859_1;
++
 +import java.io.BufferedWriter;
 +import java.io.File;
 +import java.io.FileNotFoundException;
 +import java.io.FileOutputStream;
 +import java.io.IOException;
 +import java.io.OutputStreamWriter;
 +import java.io.PrintWriter;
 +import java.net.InetAddress;
 +import java.nio.charset.Charset;
- import java.nio.charset.StandardCharsets;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Locale;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeMap;
 +import java.util.UUID;
 +import java.util.concurrent.TimeUnit;
 +
 +import jline.console.ConsoleReader;
 +import jline.console.UserInterruptException;
 +import jline.console.history.FileHistory;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.impl.ClientConfigurationHelper;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
 +import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
 +import org.apache.accumulo.core.trace.DistributedTrace;
 +import org.apache.accumulo.core.util.BadArgumentException;
 +import org.apache.accumulo.core.util.format.BinaryFormatter;
 +import org.apache.accumulo.core.util.format.DefaultFormatter;
 +import org.apache.accumulo.core.util.format.Formatter;
 +import org.apache.accumulo.core.util.format.FormatterFactory;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.start.classloader.vfs.ContextManager;
 +import org.apache.accumulo.shell.commands.AboutCommand;
 +import org.apache.accumulo.shell.commands.AddAuthsCommand;
 +import org.apache.accumulo.shell.commands.AddSplitsCommand;
 +import org.apache.accumulo.shell.commands.AuthenticateCommand;
 +import org.apache.accumulo.shell.commands.ByeCommand;
 +import org.apache.accumulo.shell.commands.ClasspathCommand;
 +import org.apache.accumulo.shell.commands.ClearCommand;
 +import org.apache.accumulo.shell.commands.CloneTableCommand;
 +import org.apache.accumulo.shell.commands.ClsCommand;
 +import org.apache.accumulo.shell.commands.CompactCommand;
 +import org.apache.accumulo.shell.commands.ConfigCommand;
 +import org.apache.accumulo.shell.commands.ConstraintCommand;
 +import org.apache.accumulo.shell.commands.CreateNamespaceCommand;
 +import org.apache.accumulo.shell.commands.CreateTableCommand;
 +import org.apache.accumulo.shell.commands.CreateUserCommand;
 +import org.apache.accumulo.shell.commands.DUCommand;
 +import org.apache.accumulo.shell.commands.DebugCommand;
 +import org.apache.accumulo.shell.commands.DeleteCommand;
 +import org.apache.accumulo.shell.commands.DeleteIterCommand;
 +import org.apache.accumulo.shell.commands.DeleteManyCommand;
 +import org.apache.accumulo.shell.commands.DeleteNamespaceCommand;
 +import org.apache.accumulo.shell.commands.DeleteRowsCommand;
 +import org.apache.accumulo.shell.commands.DeleteScanIterCommand;
 +import org.apache.accumulo.shell.commands.DeleteShellIterCommand;
 +import org.apache.accumulo.shell.commands.DeleteTableCommand;
 +import org.apache.accumulo.shell.commands.DeleteUserCommand;
 +import org.apache.accumulo.shell.commands.DropTableCommand;
 +import org.apache.accumulo.shell.commands.DropUserCommand;
 +import org.apache.accumulo.shell.commands.EGrepCommand;
 +import org.apache.accumulo.shell.commands.ExecfileCommand;
 +import org.apache.accumulo.shell.commands.ExitCommand;
 +import org.apache.accumulo.shell.commands.ExportTableCommand;
 +import org.apache.accumulo.shell.commands.ExtensionCommand;
 +import org.apache.accumulo.shell.commands.FateCommand;
 +import org.apache.accumulo.shell.commands.FlushCommand;
 +import org.apache.accumulo.shell.commands.FormatterCommand;
 +import org.apache.accumulo.shell.commands.GetAuthsCommand;
 +import org.apache.accumulo.shell.commands.GetGroupsCommand;
 +import org.apache.accumulo.shell.commands.GetSplitsCommand;
 +import org.apache.accumulo.shell.commands.GrantCommand;
 +import org.apache.accumulo.shell.commands.GrepCommand;
 +import org.apache.accumulo.shell.commands.HelpCommand;
 +import org.apache.accumulo.shell.commands.HiddenCommand;
 +import org.apache.accumulo.shell.commands.HistoryCommand;
 +import org.apache.accumulo.shell.commands.ImportDirectoryCommand;
 +import org.apache.accumulo.shell.commands.ImportTableCommand;
 +import org.apache.accumulo.shell.commands.InfoCommand;
 +import org.apache.accumulo.shell.commands.InsertCommand;
 +import org.apache.accumulo.shell.commands.InterpreterCommand;
 +import org.apache.accumulo.shell.commands.ListCompactionsCommand;
 +import org.apache.accumulo.shell.commands.ListIterCommand;
 +import org.apache.accumulo.shell.commands.ListScansCommand;
 +import org.apache.accumulo.shell.commands.ListShellIterCommand;
 +import org.apache.accumulo.shell.commands.MaxRowCommand;
 +import org.apache.accumulo.shell.commands.MergeCommand;
 +import org.apache.accumulo.shell.commands.NamespacePermissionsCommand;
 +import org.apache.accumulo.shell.commands.NamespacesCommand;
 +import org.apache.accumulo.shell.commands.NoTableCommand;
 +import org.apache.accumulo.shell.commands.OfflineCommand;
 +import org.apache.accumulo.shell.commands.OnlineCommand;
 +import org.apache.accumulo.shell.commands.OptUtil;
 +import org.apache.accumulo.shell.commands.PasswdCommand;
 +import org.apache.accumulo.shell.commands.PingCommand;
 +import org.apache.accumulo.shell.commands.QuestionCommand;
 +import org.apache.accumulo.shell.commands.QuitCommand;
 +import org.apache.accumulo.shell.commands.QuotedStringTokenizer;
 +import org.apache.accumulo.shell.commands.RenameNamespaceCommand;
 +import org.apache.accumulo.shell.commands.RenameTableCommand;
 +import org.apache.accumulo.shell.commands.RevokeCommand;
 +import org.apache.accumulo.shell.commands.ScanCommand;
 +import org.apache.accumulo.shell.commands.ScriptCommand;
 +import org.apache.accumulo.shell.commands.SetAuthsCommand;
 +import org.apache.accumulo.shell.commands.SetGroupsCommand;
 +import org.apache.accumulo.shell.commands.SetIterCommand;
 +import org.apache.accumulo.shell.commands.SetScanIterCommand;
 +import org.apache.accumulo.shell.commands.SetShellIterCommand;
 +import org.apache.accumulo.shell.commands.SleepCommand;
 +import org.apache.accumulo.shell.commands.SystemPermissionsCommand;
 +import org.apache.accumulo.shell.commands.TableCommand;
 +import org.apache.accumulo.shell.commands.TablePermissionsCommand;
 +import org.apache.accumulo.shell.commands.TablesCommand;
 +import org.apache.accumulo.shell.commands.TraceCommand;
 +import org.apache.accumulo.shell.commands.UserCommand;
 +import org.apache.accumulo.shell.commands.UserPermissionsCommand;
 +import org.apache.accumulo.shell.commands.UsersCommand;
 +import org.apache.accumulo.shell.commands.WhoAmICommand;
 +import org.apache.commons.cli.BasicParser;
 +import org.apache.commons.cli.CommandLine;
 +import org.apache.commons.cli.HelpFormatter;
 +import org.apache.commons.cli.MissingOptionException;
 +import org.apache.commons.cli.Option;
 +import org.apache.commons.cli.Options;
 +import org.apache.commons.cli.ParseException;
 +import org.apache.commons.vfs2.FileSystemException;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.JCommander;
 +import com.beust.jcommander.ParameterException;
 +
 +/**
 + * A convenient console interface to perform basic accumulo functions Includes auto-complete, help, and quoted strings with escape sequences
 + */
 +public class Shell extends ShellOptions {
 +  public static final Logger log = Logger.getLogger(Shell.class);
 +  private static final Logger audit = Logger.getLogger(Shell.class.getName() + ".audit");
 +
-   public static final Charset CHARSET = StandardCharsets.ISO_8859_1;
++  public static final Charset CHARSET = ISO_8859_1;
 +  public static final int NO_FIXED_ARG_LENGTH_CHECK = -1;
 +  public static final String COMMENT_PREFIX = "#";
 +  public static final String HISTORY_DIR_NAME = ".accumulo";
 +  public static final String HISTORY_FILE_NAME = "shell_history.txt";
 +  private static final String SHELL_DESCRIPTION = "Shell - Apache Accumulo Interactive Shell";
 +
 +  protected int exitCode = 0;
 +  private String tableName;
 +  protected Instance instance;
 +  private Connector connector;
 +  protected ConsoleReader reader;
 +  private String principal;
 +  private AuthenticationToken token;
 +  private final Class<? extends Formatter> defaultFormatterClass = DefaultFormatter.class;
 +  private final Class<? extends Formatter> binaryFormatterClass = BinaryFormatter.class;
 +  public Map<String,List<IteratorSetting>> scanIteratorOptions = new HashMap<String,List<IteratorSetting>>();
 +  public Map<String,List<IteratorSetting>> iteratorProfiles = new HashMap<String,List<IteratorSetting>>();
 +
 +  private Token rootToken;
 +  public final Map<String,Command> commandFactory = new TreeMap<String,Command>();
 +  public final Map<String,Command[]> commandGrouping = new TreeMap<String,Command[]>();
 +  protected boolean configError = false;
 +
 +  // exit if true
 +  private boolean exit = false;
 +
 +  // file to execute commands from
 +  protected File execFile = null;
 +  // single command to execute from the command line
 +  protected String execCommand = null;
 +  protected boolean verbose = true;
 +
 +  private boolean tabCompletion;
 +  private boolean disableAuthTimeout;
 +  private long authTimeout;
 +  private long lastUserActivity = System.nanoTime();
 +  private boolean logErrorsToConsole = false;
 +  private PrintWriter writer = null;
 +  private boolean masking = false;
 +
 +  public Shell() throws IOException {
 +    this(new ConsoleReader(), new PrintWriter(
 +        new OutputStreamWriter(System.out,
 +        System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
 +  }
 +
 +  public Shell(ConsoleReader reader, PrintWriter writer) {
 +    super();
 +    this.reader = reader;
 +    this.writer = writer;
 +  }
 +
 +  // Not for client use
 +  public boolean config(String... args) {
 +    ShellOptionsJC options = new ShellOptionsJC();
 +    JCommander jc = new JCommander();
 +
 +    jc.setProgramName("accumulo shell");
 +    jc.addObject(options);
 +    try {
 +      jc.parse(args);
 +    } catch (ParameterException e) {
 +      configError = true;
 +    }
 +
 +    if (options.isHelpEnabled()) {
 +      configError = true;
 +    }
 +
 +    if (!configError && options.getUnrecognizedOptions() != null) {
 +      configError = true;
 +      logError("Unrecognized Options: " + options.getUnrecognizedOptions().toString());
 +    }
 +
 +    if (configError) {
 +      jc.usage();
 +      return true;
 +    }
 +
 +    setDebugging(options.isDebugEnabled());
 +    authTimeout = TimeUnit.MINUTES.toNanos(options.getAuthTimeout());
 +    disableAuthTimeout = options.isAuthTimeoutDisabled();
 +
 +    // get the options that were parsed
 +    String user = options.getUsername();
 +    String password = options.getPassword();
 +
 +    tabCompletion = !options.isTabCompletionDisabled();
 +
 +    // Use a fake (Mock), ZK, or HdfsZK Accumulo instance
 +    setInstance(options);
 +
 +    // AuthenticationToken options
 +    token = options.getAuthenticationToken();
 +    Map<String,String> loginOptions = options.getTokenProperties();
 +
 +    // process default parameters if unspecified
 +    try {
 +      boolean hasToken = (token != null);
 +      boolean hasTokenOptions = !loginOptions.isEmpty();
 +
 +      if (hasToken && password != null) {
 +        throw new ParameterException("Can not supply '--pass' option with '--tokenClass' option");
 +      }
 +
 +      Runtime.getRuntime().addShutdownHook(new Thread() {
 +        @Override
 +        public void run() {
 +          reader.getTerminal().setEchoEnabled(true);
 +        }
 +      });
 +
 +      // Need either both a token and options, or neither, but not just one.
 +      if (hasToken != hasTokenOptions) {
 +        throw new ParameterException("Must supply either both or neither of '--tokenClass' and '--tokenProperty'");
 +      } else if (hasToken) { // implied hasTokenOptions
 +        // Fully qualified name so we don't shadow java.util.Properties
 +        org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties props;
 +        // and line wrap it because the package name is so long
 +        props = new org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties();
 +
 +        props.putAllStrings(loginOptions);
 +        token.init(props);
 +      } else {
 +        // Read password if the user explicitly asked for it, or didn't specify anything at all
 +        if ("stdin".equals(password) || password == null) {
 +          password = reader.readLine("Password: ", '*');
 +        }
 +
 +        if (password == null) {
 +          // User cancel, e.g. Ctrl-D pressed
 +          throw new ParameterException("No password or token option supplied");
 +        } else {
 +          this.token = new PasswordToken(password);
 +        }
 +      }
 +
 +      if (!options.isFake()) {
 +        ZooReader zr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +        DistributedTrace.enable(instance, zr, "shell", InetAddress.getLocalHost().getHostName());
 +      }
 +
 +      this.setTableName("");
 +      this.principal = user;
 +      connector = instance.getConnector(this.principal, token);
 +
 +    } catch (Exception e) {
 +      printException(e);
 +      configError = true;
 +    }
 +
 +    // decide whether to execute commands from a file and quit
 +    if (options.getExecFile() != null) {
 +      execFile = options.getExecFile();
 +      verbose = false;
 +    } else if (options.getExecFileVerbose() != null) {
 +      execFile = options.getExecFileVerbose();
 +      verbose = true;
 +    }
 +    execCommand = options.getExecCommand();
 +    if (execCommand != null) {
 +      verbose = false;
 +    }
 +
 +    rootToken = new Token();
 +
 +    Command[] dataCommands = {new DeleteCommand(), new DeleteManyCommand(), new DeleteRowsCommand(), new EGrepCommand(), new FormatterCommand(),
 +        new InterpreterCommand(), new GrepCommand(), new ImportDirectoryCommand(), new InsertCommand(), new MaxRowCommand(), new ScanCommand()};
 +    Command[] debuggingCommands = {new ClasspathCommand(), new DebugCommand(), new ListScansCommand(), new ListCompactionsCommand(), new TraceCommand(),
 +        new PingCommand()};
 +    Command[] execCommands = {new ExecfileCommand(), new HistoryCommand(), new ExtensionCommand(), new ScriptCommand()};
 +    Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new QuitCommand()};
 +    Command[] helpCommands = {new AboutCommand(), new HelpCommand(), new InfoCommand(), new QuestionCommand()};
 +    Command[] iteratorCommands = {new DeleteIterCommand(), new DeleteScanIterCommand(), new ListIterCommand(), new SetIterCommand(), new SetScanIterCommand(),
 +        new SetShellIterCommand(), new ListShellIterCommand(), new DeleteShellIterCommand()};
 +    Command[] otherCommands = {new HiddenCommand()};
 +    Command[] permissionsCommands = {new GrantCommand(), new RevokeCommand(), new SystemPermissionsCommand(), new TablePermissionsCommand(),
 +        new UserPermissionsCommand(), new NamespacePermissionsCommand()};
 +    Command[] stateCommands = {new AuthenticateCommand(), new ClsCommand(), new ClearCommand(), new FateCommand(), new NoTableCommand(), new SleepCommand(),
 +        new TableCommand(), new UserCommand(), new WhoAmICommand()};
 +    Command[] tableCommands = {new CloneTableCommand(), new ConfigCommand(), new CreateTableCommand(), new DeleteTableCommand(), new DropTableCommand(),
 +        new DUCommand(), new ExportTableCommand(), new ImportTableCommand(), new OfflineCommand(), new OnlineCommand(), new RenameTableCommand(),
 +        new TablesCommand(), new NamespacesCommand(), new CreateNamespaceCommand(), new DeleteNamespaceCommand(), new RenameNamespaceCommand()};
 +    Command[] tableControlCommands = {new AddSplitsCommand(), new CompactCommand(), new ConstraintCommand(), new FlushCommand(), new GetGroupsCommand(),
 +        new GetSplitsCommand(), new MergeCommand(), new SetGroupsCommand()};
 +    Command[] userCommands = {new AddAuthsCommand(), new CreateUserCommand(), new DeleteUserCommand(), new DropUserCommand(), new GetAuthsCommand(),
 +        new PasswdCommand(), new SetAuthsCommand(), new UsersCommand()};
 +    commandGrouping.put("-- Writing, Reading, and Removing Data --", dataCommands);
 +    commandGrouping.put("-- Debugging Commands -------------------", debuggingCommands);
 +    commandGrouping.put("-- Shell Execution Commands -------------", execCommands);
 +    commandGrouping.put("-- Exiting Commands ---------------------", exitCommands);
 +    commandGrouping.put("-- Help Commands ------------------------", helpCommands);
 +    commandGrouping.put("-- Iterator Configuration ---------------", iteratorCommands);
 +    commandGrouping.put("-- Permissions Administration Commands --", permissionsCommands);
 +    commandGrouping.put("-- Shell State Commands -----------------", stateCommands);
 +    commandGrouping.put("-- Table Administration Commands --------", tableCommands);
 +    commandGrouping.put("-- Table Control Commands ---------------", tableControlCommands);
 +    commandGrouping.put("-- User Administration Commands ---------", userCommands);
 +
 +    for (Command[] cmds : commandGrouping.values()) {
 +      for (Command cmd : cmds)
 +        commandFactory.put(cmd.getName(), cmd);
 +    }
 +    for (Command cmd : otherCommands) {
 +      commandFactory.put(cmd.getName(), cmd);
 +    }
 +    return configError;
 +  }
 +
 +  /**
 +   * Sets the instance used by the shell based on the given options.
 +   * 
 +   * @param options
 +   *          shell options
 +   */
 +  protected void setInstance(ShellOptionsJC options) {
 +    // should only be one set of instance options set
 +    instance = null;
 +    if (options.isFake()) {
 +      instance = new MockInstance("fake");
 +    } else {
 +      String instanceName, hosts;
 +      if (options.isHdfsZooInstance()) {
 +        instanceName = hosts = null;
 +      } else if (options.getZooKeeperInstance().size() > 0) {
 +        List<String> zkOpts = options.getZooKeeperInstance();
 +        instanceName = zkOpts.get(0);
 +        hosts = zkOpts.get(1);
 +      } else {
 +        instanceName = options.getZooKeeperInstanceName();
 +        hosts = options.getZooKeeperHosts();
 +      }
 +      try {
 +        instance = getZooInstance(instanceName, hosts, options.getClientConfiguration());
 +      } catch (Exception e) {
 +        throw new IllegalArgumentException("Unable to load client config from " + options.getClientConfigFile(), e);
 +      }
 +    }
 +  }
 +
 +  /*
 +   * Takes instanceName and keepers as separate arguments, rather than just packaged into the clientConfig, so that we can fail over to accumulo-site.xml or
 +   * HDFS config if they're unspecified.
 +   */
 +  private static Instance getZooInstance(String instanceName, String keepers, ClientConfiguration clientConfig) {
 +    UUID instanceId = null;
 +    if (instanceName == null) {
 +      instanceName = clientConfig.get(ClientProperty.INSTANCE_NAME);
 +    }
 +    if (instanceName == null || keepers == null) {
 +      AccumuloConfiguration conf = SiteConfiguration.getInstance(ClientConfigurationHelper.convertClientConfig(DefaultConfiguration.getInstance(), clientConfig));
 +      if (instanceName == null) {
 +        Path instanceDir = new Path(VolumeConfiguration.getVolumeUris(conf)[0], "instance_id");
 +        instanceId = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(instanceDir, conf));
 +      }
 +      if (keepers == null) {
 +        keepers = conf.get(Property.INSTANCE_ZK_HOST);
 +      }
 +    }
 +    if (instanceId != null) {
 +      return new ZooKeeperInstance(clientConfig.withInstance(instanceId).withZkHosts(keepers));
 +    } else {
 +      return new ZooKeeperInstance(clientConfig.withInstance(instanceName).withZkHosts(keepers));
 +    }
 +  }
 +
 +  public Connector getConnector() {
 +    return connector;
 +  }
 +
 +  public Instance getInstance() {
 +    return instance;
 +  }
 +
 +  public ClassLoader getClassLoader(final CommandLine cl, final Shell shellState) throws AccumuloException, TableNotFoundException, AccumuloSecurityException,
 +      IOException, FileSystemException {
 +
 +    boolean tables = cl.hasOption(OptUtil.tableOpt().getOpt()) || !shellState.getTableName().isEmpty();
 +    boolean namespaces = cl.hasOption(OptUtil.namespaceOpt().getOpt());
 +
 +    String classpath = null;
 +    Iterable<Entry<String,String>> tableProps;
 +
 +    if (namespaces) {
 +      try {
 +        tableProps = shellState.getConnector().namespaceOperations().getProperties(OptUtil.getNamespaceOpt(cl, shellState));
 +      } catch (NamespaceNotFoundException e) {
 +        throw new IllegalArgumentException(e);
 +      }
 +    } else if (tables) {
 +      tableProps = shellState.getConnector().tableOperations().getProperties(OptUtil.getTableOpt(cl, shellState));
 +    } else {
 +      throw new IllegalArgumentException("No table or namespace specified");
 +    }
 +    for (Entry<String,String> entry : tableProps) {
 +      if (entry.getKey().equals(Property.TABLE_CLASSPATH.getKey())) {
 +        classpath = entry.getValue();
 +      }
 +    }
 +
 +    ClassLoader classloader;
 +
 +    if (classpath != null && !classpath.equals("")) {
 +      shellState.getConnector().instanceOperations().getSystemConfiguration().get(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + classpath);
 +
 +      try {
 +        AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Map.Entry<String,String>>() {
 +          @Override
 +          public Iterator<Entry<String,String>> iterator() {
 +            try {
 +              return shellState.getConnector().instanceOperations().getSystemConfiguration().entrySet().iterator();
 +            } catch (AccumuloException e) {
 +              throw new RuntimeException(e);
 +            } catch (AccumuloSecurityException e) {
 +              throw new RuntimeException(e);
 +            }
 +          }
 +        }));
 +      } catch (IllegalStateException ise) {}
 +
 +      classloader = AccumuloVFSClassLoader.getContextManager().getClassLoader(classpath);
 +    } else {
 +      classloader = AccumuloVFSClassLoader.getClassLoader();
 +    }
 +    return classloader;
 +  }
 +
 +  public static void main(String args[]) throws IOException {
 +    Shell shell = new Shell();
 +    try {
 +      shell.config(args);
 +
 +      System.exit(shell.start());
 +    } finally {
 +      shell.shutdown();
 +    }
 +  }
 +
 +  public int start() throws IOException {
 +    if (configError)
 +      return 1;
 +
 +    String input;
 +    if (isVerbose())
 +      printInfo();
 +
 +    String home = System.getProperty("HOME");
 +    if (home == null)
 +      home = System.getenv("HOME");
 +    String configDir = home + "/" + HISTORY_DIR_NAME;
 +    String historyPath = configDir + "/" + HISTORY_FILE_NAME;
 +    File accumuloDir = new File(configDir);
 +    if (!accumuloDir.exists() && !accumuloDir.mkdirs())
 +      log.warn("Unable to make directory for history at " + accumuloDir);
 +    try {
 +      final FileHistory history = new FileHistory(new File(historyPath));
 +      reader.setHistory(history);
 +      // Add shutdown hook to flush file history, per jline javadocs
 +      Runtime.getRuntime().addShutdownHook(new Thread() {
 +        @Override
 +        public void run() {
 +          try {
 +            history.flush();
 +          } catch (IOException e) {
 +            log.warn("Could not flush history to file.");
 +          }
 +        }
 +      });
 +    } catch (IOException e) {
 +      log.warn("Unable to load history file at " + historyPath);
 +    }
 +
 +    // Turn Ctrl+C into Exception instead of JVM exit
 +    reader.setHandleUserInterrupt(true);
 +
 +    ShellCompletor userCompletor = null;
 +
 +    if (execFile != null) {
-       java.util.Scanner scanner = new java.util.Scanner(execFile, StandardCharsets.UTF_8.name());
++      java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
 +      try {
 +        while (scanner.hasNextLine() && !hasExited()) {
 +          execCommand(scanner.nextLine(), true, isVerbose());
 +        }
 +      } finally {
 +        scanner.close();
 +      }
 +    } else if (execCommand != null) {
 +      for (String command : execCommand.split("\n")) {
 +        execCommand(command, true, isVerbose());
 +      }
 +      return exitCode;
 +    }
 +
 +    while (true) {
 +      try {
 +        if (hasExited())
 +          return exitCode;
 +
 +        // If tab completion is true we need to reset
 +        if (tabCompletion) {
 +          if (userCompletor != null)
 +            reader.removeCompleter(userCompletor);
 +
 +          userCompletor = setupCompletion();
 +          reader.addCompleter(userCompletor);
 +        }
 +
 +        reader.setPrompt(getDefaultPrompt());
 +        input = reader.readLine();
 +        if (input == null) {
 +          reader.println();
 +          return exitCode;
 +        } // User Canceled (Ctrl+D)
 +
 +        execCommand(input, disableAuthTimeout, false);
 +      } catch (UserInterruptException uie) {
 +        // User Cancelled (Ctrl+C)
 +        reader.println();
 +
 +        String partialLine = uie.getPartialLine();
 +        if (partialLine == null || "".equals(uie.getPartialLine().trim())) {
 +          // No content, actually exit
 +          return exitCode;
 +        }
 +      } finally {
 +        reader.flush();
 +      }
 +    }
 +  }
 +
 +  public void shutdown() {
 +    if (reader != null) {
 +      reader.shutdown();
 +    }
 +  }
 +
 +  public void printInfo() throws IOException {
 +    reader.print("\n" + SHELL_DESCRIPTION + "\n" + "- \n" + "- version: " + Constants.VERSION + "\n" + "- instance name: "
 +        + connector.getInstance().getInstanceName() + "\n" + "- instance id: " + connector.getInstance().getInstanceID() + "\n" + "- \n"
 +        + "- type 'help' for a list of available commands\n" + "- \n");
 +    reader.flush();
 +  }
 +
 +  public void printVerboseInfo() throws IOException {
 +    StringBuilder sb = new StringBuilder("-\n");
 +    sb.append("- Current user: ").append(connector.whoami()).append("\n");
 +    if (execFile != null)
 +      sb.append("- Executing commands from: ").append(execFile).append("\n");
 +    if (disableAuthTimeout)
 +      sb.append("- Authorization timeout: disabled\n");
 +    else
 +      sb.append("- Authorization timeout: ").append(String.format("%ds%n", TimeUnit.NANOSECONDS.toSeconds(authTimeout)));
 +    sb.append("- Debug: ").append(isDebuggingEnabled() ? "on" : "off").append("\n");
 +    if (!scanIteratorOptions.isEmpty()) {
 +      for (Entry<String,List<IteratorSetting>> entry : scanIteratorOptions.entrySet()) {
 +        sb.append("- Session scan iterators for table ").append(entry.getKey()).append(":\n");
 +        for (IteratorSetting setting : entry.getValue()) {
 +          sb.append("-    Iterator ").append(setting.getName()).append(" options:\n");
 +          sb.append("-        ").append("iteratorPriority").append(" = ").append(setting.getPriority()).append("\n");
 +          sb.append("-        ").append("iteratorClassName").append(" = ").append(setting.getIteratorClass()).append("\n");
 +          for (Entry<String,String> optEntry : setting.getOptions().entrySet()) {
 +            sb.append("-        ").append(optEntry.getKey()).append(" = ").append(optEntry.getValue()).append("\n");
 +          }
 +        }
 +      }
 +    }
 +    sb.append("-\n");
 +    reader.print(sb.toString());
 +  }
 +
 +  public String getDefaultPrompt() {
 +    return connector.whoami() + "@" + connector.getInstance().getInstanceName() + (getTableName().isEmpty() ? "" : " ") + getTableName() + "> ";
 +  }
 +
 +  public void execCommand(String input, boolean ignoreAuthTimeout, boolean echoPrompt) throws IOException {
 +    audit.log(Level.INFO, getDefaultPrompt() + input);
 +    if (echoPrompt) {
 +      reader.print(getDefaultPrompt());
 +      reader.println(input);
 +    }
 +
 +    if (input.startsWith(COMMENT_PREFIX)) {
 +      return;
 +    }
 +
 +    String fields[];
 +    try {
 +      fields = new QuotedStringTokenizer(input).getTokens();
 +    } catch (BadArgumentException e) {
 +      printException(e);
 +      ++exitCode;
 +      return;
 +    }
 +    if (fields.length == 0)
 +      return;
 +
 +    String command = fields[0];
 +    fields = fields.length > 1 ? Arrays.copyOfRange(fields, 1, fields.length) : new String[] {};
 +
 +    Command sc = null;
 +    if (command.length() > 0) {
 +      try {
 +        // Obtain the command from the command table
 +        sc = commandFactory.get(command);
 +        if (sc == null) {
 +          reader.println(String.format("Unknown command \"%s\".  Enter \"help\" for a list possible commands.", command));
 +          reader.flush();
 +          return;
 +        }
 +
 +        long duration = System.nanoTime() - lastUserActivity;
 +        if (!(sc instanceof ExitCommand) && !ignoreAuthTimeout && (duration < 0 || duration > authTimeout)) {
 +          reader.println("Shell has been idle for too long. Please re-authenticate.");
 +          boolean authFailed = true;
 +          do {
 +            String pwd = readMaskedLine("Enter current password for '" + connector.whoami() + "': ", '*');
 +            if (pwd == null) {
 +              reader.println();
 +              return;
 +            } // user canceled
 +
 +            try {
 +              authFailed = !connector.securityOperations().authenticateUser(connector.whoami(), new PasswordToken(pwd));
 +            } catch (Exception e) {
 +              ++exitCode;
 +              printException(e);
 +            }
 +
 +            if (authFailed)
 +              reader.print("Invalid password. ");
 +          } while (authFailed);
 +          lastUserActivity = System.nanoTime();
 +        }
 +
 +        // Get the options from the command on how to parse the string
 +        Options parseOpts = sc.getOptionsWithHelp();
 +
 +        // Parse the string using the given options
 +        CommandLine cl = new BasicParser().parse(parseOpts, fields);
 +
 +        int actualArgLen = cl.getArgs().length;
 +        int expectedArgLen = sc.numArgs();
 +        if (cl.hasOption(helpOption)) {
 +          // Display help if asked to; otherwise execute the command
 +          sc.printHelp(this);
 +        } else if (expectedArgLen != NO_FIXED_ARG_LENGTH_CHECK && actualArgLen != expectedArgLen) {
 +          ++exitCode;
 +          // Check for valid number of fixed arguments (if not
 +          // negative; negative means it is not checked, for
 +          // vararg-like commands)
 +          printException(new IllegalArgumentException(String.format("Expected %d argument%s. There %s %d.", expectedArgLen, expectedArgLen == 1 ? "" : "s",
 +              actualArgLen == 1 ? "was" : "were", actualArgLen)));
 +          sc.printHelp(this);
 +        } else {
 +          int tmpCode = sc.execute(input, cl, this);
 +          exitCode += tmpCode;
 +          reader.flush();
 +        }
 +
 +      } catch (ConstraintViolationException e) {
 +        ++exitCode;
 +        printConstraintViolationException(e);
 +      } catch (TableNotFoundException e) {
 +        ++exitCode;
 +        if (getTableName().equals(e.getTableName()))
 +          setTableName("");
 +        printException(e);
 +      } catch (ParseException e) {
 +        // not really an error if the exception is a missing required
 +        // option when the user is asking for help
 +        if (!(e instanceof MissingOptionException && (Arrays.asList(fields).contains("-" + helpOption) || Arrays.asList(fields).contains("--" + helpLongOption)))) {
 +          ++exitCode;
 +          printException(e);
 +        }
 +        if (sc != null)
 +          sc.printHelp(this);
 +      } catch (UserInterruptException e) {
 +        ++exitCode;
 +      } catch (Exception e) {
 +        ++exitCode;
 +        printException(e);
 +      }
 +    } else {
 +      ++exitCode;
 +      printException(new BadArgumentException("Unrecognized empty command", command, -1));
 +    }
 +    reader.flush();
 +  }
 +
 +  /**
 +   * The command tree is built in reverse so that the references are more easily linked up. There is some code in token to allow forward building of the command
 +   * tree.
 +   */
 +  private ShellCompletor setupCompletion() {
 +    rootToken = new Token();
 +
 +    Set<String> tableNames = null;
 +    try {
 +      tableNames = connector.tableOperations().list();
 +    } catch (Exception e) {
 +      log.debug("Unable to obtain list of tables", e);
 +      tableNames = Collections.emptySet();
 +    }
 +
 +    Set<String> userlist = null;
 +    try {
 +      userlist = connector.securityOperations().listLocalUsers();
 +    } catch (Exception e) {
 +      log.debug("Unable to obtain list of users", e);
 +      userlist = Collections.emptySet();
 +    }
 +
 +    Set<String> namespaces = null;
 +    try {
 +      namespaces = connector.namespaceOperations().list();
 +    } catch (Exception e) {
 +      log.debug("Unable to obtain list of namespaces", e);
 +      namespaces = Collections.emptySet();
 +    }
 +
 +    Map<Command.CompletionSet,Set<String>> options = new HashMap<Command.CompletionSet,Set<String>>();
 +
 +    Set<String> commands = new HashSet<String>();
 +    for (String a : commandFactory.keySet())
 +      commands.add(a);
 +
 +    Set<String> modifiedUserlist = new HashSet<String>();
 +    Set<String> modifiedTablenames = new HashSet<String>();
 +    Set<String> modifiedNamespaces = new HashSet<String>();
 +
 +    for (String a : tableNames)
 +      modifiedTablenames.add(a.replaceAll("([\\s'\"])", "\\\\$1"));
 +    for (String a : userlist)
 +      modifiedUserlist.add(a.replaceAll("([\\s'\"])", "\\\\$1"));
 +    for (String a : namespaces) {
 +      String b = a.replaceAll("([\\s'\"])", "\\\\$1");
 +      modifiedNamespaces.add(b.isEmpty() ? "\"\"" : b);
 +    }
 +
 +    options.put(Command.CompletionSet.USERNAMES, modifiedUserlist);
 +    options.put(Command.CompletionSet.TABLENAMES, modifiedTablenames);
 +    options.put(Command.CompletionSet.NAMESPACES, modifiedNamespaces);
 +    options.put(Command.CompletionSet.COMMANDS, commands);
 +
 +    for (Command[] cmdGroup : commandGrouping.values()) {
 +      for (Command c : cmdGroup) {
 +        c.getOptionsWithHelp(); // prep the options for the command
 +        // so that the completion can
 +        // include them
 +        c.registerCompletion(rootToken, options);
 +      }
 +    }
 +    return new ShellCompletor(rootToken, options);
 +  }
 +
 +  /**
 +   * The Command class represents a command to be run in the shell. It contains the methods to execute along with some methods to help tab completion, and
 +   * return the command name, help, and usage.
 +   */
 +  public static abstract class Command {
 +    // Helper methods for completion
 +    public enum CompletionSet {
 +      TABLENAMES, USERNAMES, COMMANDS, NAMESPACES
 +    }
 +
 +    static Set<String> getCommandNames(Map<CompletionSet,Set<String>> objects) {
 +      return objects.get(CompletionSet.COMMANDS);
 +    }
 +
 +    static Set<String> getTableNames(Map<CompletionSet,Set<String>> objects) {
 +      return objects.get(CompletionSet.TABLENAMES);
 +    }
 +
 +    static Set<String> getUserNames(Map<CompletionSet,Set<String>> objects) {
 +      return objects.get(CompletionSet.USERNAMES);
 +    }
 +
 +    static Set<String> getNamespaces(Map<CompletionSet,Set<String>> objects) {
 +      return objects.get(CompletionSet.NAMESPACES);
 +    }
 +
 +    public void registerCompletionGeneral(Token root, Set<String> args, boolean caseSens) {
 +      Token t = new Token(args);
 +      t.setCaseSensitive(caseSens);
 +
 +      Token command = new Token(getName());
 +      command.addSubcommand(t);
 +
 +      root.addSubcommand(command);
 +    }
 +
 +    public void registerCompletionForTables(Token root, Map<CompletionSet,Set<String>> completionSet) {
 +      registerCompletionGeneral(root, completionSet.get(CompletionSet.TABLENAMES), true);
 +    }
 +
 +    public void registerCompletionForUsers(Token root, Map<CompletionSet,Set<String>> completionSet) {
 +      registerCompletionGeneral(root, completionSet.get(CompletionSet.USERNAMES), true);
 +    }
 +
 +    public void registerCompletionForCommands(Token root, Map<CompletionSet,Set<String>> completionSet) {
 +      registerCompletionGeneral(root, completionSet.get(CompletionSet.COMMANDS), false);
 +    }
 +
 +    public void registerCompletionForNamespaces(Token root, Map<CompletionSet,Set<String>> completionSet) {
 +      registerCompletionGeneral(root, completionSet.get(CompletionSet.NAMESPACES), true);
 +    }
 +
 +    // abstract methods to override
 +    public abstract int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception;
 +
 +    public abstract String description();
 +
 +    /**
 +     * If the number of arguments is not always zero (not including those arguments handled through Options), make sure to override the {@link #usage()} method.
 +     * Otherwise, {@link #usage()} does need to be overridden.
 +     */
 +    public abstract int numArgs();
 +
 +    // OPTIONAL methods to override:
 +
 +    // the general version of getname uses reflection to get the class name
 +    // and then cuts off the suffix -Command to get the name of the command
 +    public String getName() {
 +      String s = this.getClass().getName();
 +      int st = Math.max(s.lastIndexOf('$'), s.lastIndexOf('.'));
 +      int i = s.indexOf("Command");
 +      return i > 0 ? s.substring(st + 1, i).toLowerCase(Locale.ENGLISH) : null;
 +    }
 +
 +    // The general version of this method adds the name
 +    // of the command to the completion tree
 +    public void registerCompletion(Token root, Map<CompletionSet,Set<String>> completion_set) {
 +      root.addSubcommand(new Token(getName()));
 +    }
 +
 +    // The general version of this method uses the HelpFormatter
 +    // that comes with the apache Options package to print out the help
 +    public final void printHelp(Shell shellState) {
 +      shellState.printHelp(usage(), "description: " + this.description(), getOptionsWithHelp());
 +    }
 +
 +    public final void printHelp(Shell shellState, int width) {
 +      shellState.printHelp(usage(), "description: " + this.description(), getOptionsWithHelp(), width);
 +    }
 +
 +    // Get options with help
 +    public final Options getOptionsWithHelp() {
 +      Options opts = getOptions();
 +      opts.addOption(new Option(helpOption, helpLongOption, false, "display this help"));
 +      return opts;
 +    }
 +
 +    // General usage is just the command
 +    public String usage() {
 +      return getName();
 +    }
 +
 +    // General Options are empty
 +    public Options getOptions() {
 +      return new Options();
 +    }
 +  }
 +
 +  public interface PrintLine {
 +    void print(String s);
 +
 +    void close();
 +  }
 +
 +  public static class PrintShell implements PrintLine {
 +    ConsoleReader reader;
 +
 +    public PrintShell(ConsoleReader reader) {
 +      this.reader = reader;
 +    }
 +
 +    @Override
 +    public void print(String s) {
 +      try {
 +        reader.println(s);
 +      } catch (Exception ex) {
 +        throw new RuntimeException(ex);
 +      }
 +    }
 +
 +    @Override
 +    public void close() {}
 +  };
 +
 +  public static class PrintFile implements PrintLine {
 +    PrintWriter writer;
 +
 +    public PrintFile(String filename) throws FileNotFoundException {
-       writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), StandardCharsets.UTF_8)));
++      writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), UTF_8)));
 +    }
 +
 +    @Override
 +    public void print(String s) {
 +      writer.println(s);
 +    }
 +
 +    @Override
 +    public void close() {
 +      writer.close();
 +    }
 +  };
 +
 +  public final void printLines(Iterator<String> lines, boolean paginate) throws IOException {
 +    printLines(lines, paginate, null);
 +  }
 +
 +  public final void printLines(Iterator<String> lines, boolean paginate, PrintLine out) throws IOException {
 +    int linesPrinted = 0;
 +    String prompt = "-- hit any key to continue or 'q' to quit --";
 +    int lastPromptLength = prompt.length();
 +    int termWidth = reader.getTerminal().getWidth();
 +    int maxLines = reader.getTerminal().getHeight();
 +
 +    String peek = null;
 +    while (lines.hasNext()) {
 +      String nextLine = lines.next();
 +      if (nextLine == null)
 +        continue;
 +      for (String line : nextLine.split("\\n")) {
 +        if (out == null) {
 +          if (peek != null) {
 +            reader.println(peek);
 +            if (paginate) {
 +              linesPrinted += peek.length() == 0 ? 0 : Math.ceil(peek.length() * 1.0 / termWidth);
 +
 +              // check if displaying the next line would result in
 +              // scrolling off the screen
 +              if (linesPrinted + Math.ceil(lastPromptLength * 1.0 / termWidth) + Math.ceil(prompt.length() * 1.0 / termWidth)
 +                  + Math.ceil(line.length() * 1.0 / termWidth) > maxLines) {
 +                linesPrinted = 0;
 +                int numdashes = (termWidth - prompt.length()) / 2;
 +                String nextPrompt = repeat("-", numdashes) + prompt + repeat("-", numdashes);
 +                lastPromptLength = nextPrompt.length();
 +                reader.print(nextPrompt);
 +                reader.flush();
 +
 +                if (Character.toUpperCase((char) reader.readCharacter()) == 'Q') {
 +                  reader.println();
 +                  return;
 +                }
 +                reader.println();
 +                termWidth = reader.getTerminal().getWidth();
 +                maxLines = reader.getTerminal().getHeight();
 +              }
 +            }
 +          }
 +          peek = line;
 +        } else {
 +          out.print(line);
 +        }
 +      }
 +    }
 +    if (out == null && peek != null) {
 +      reader.println(peek);
 +    }
 +  }
 +
 +  public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, Class<? extends Formatter> formatterClass,
 +      PrintLine outFile) throws IOException {
 +    printLines(FormatterFactory.getFormatter(formatterClass, scanner, printTimestamps), paginate, outFile);
 +  }
 +
 +  public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, Class<? extends Formatter> formatterClass)
 +      throws IOException {
 +    printLines(FormatterFactory.getFormatter(formatterClass, scanner, printTimestamps), paginate);
 +  }
 +
 +  public final void printBinaryRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, PrintLine outFile) throws IOException {
 +    printLines(FormatterFactory.getFormatter(binaryFormatterClass, scanner, printTimestamps), paginate, outFile);
 +  }
 +
 +  public final void printBinaryRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate) throws IOException {
 +    printLines(FormatterFactory.getFormatter(binaryFormatterClass, scanner, printTimestamps), paginate);
 +  }
 +
 +  public static String repeat(String s, int c) {
 +    StringBuilder sb = new StringBuilder();
 +    for (int i = 0; i < c; i++)
 +      sb.append(s);
 +    return sb.toString();
 +  }
 +
 +  public void checkTableState() {
 +    if (getTableName().isEmpty())
 +      throw new IllegalStateException(
 +          "Not in a table context. Please use 'table <tableName>' to switch to a table, or use '-t' to specify a table if option is available.");
 +  }
 +
 +  private final void printConstraintViolationException(ConstraintViolationException cve) {
 +    printException(cve, "");
 +    int COL1 = 50, COL2 = 14;
 +    int col3 = Math.max(1, Math.min(Integer.MAX_VALUE, reader.getTerminal().getWidth() - COL1 - COL2 - 6));
 +    logError(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s%n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
 +    logError(String.format("%-" + COL1 + "s | %" + COL2 + "s | %-" + col3 + "s%n", "Constraint class", "Violation code", "Violation Description"));
 +    logError(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s%n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
 +    for (TConstraintViolationSummary cvs : cve.violationSummaries)
 +      logError(String.format("%-" + COL1 + "s | %" + COL2 + "d | %-" + col3 + "s%n", cvs.constrainClass, cvs.violationCode, cvs.violationDescription));
 +    logError(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s%n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
 +  }
 +
 +  public final void printException(Exception e) {
 +    printException(e, e.getMessage());
 +  }
 +
 +  private final void printException(Exception e, String msg) {
 +    logError(e.getClass().getName() + (msg != null ? ": " + msg : ""));
 +    log.debug(e.getClass().getName() + (msg != null ? ": " + msg : ""), e);
 +  }
 +
 +  public static final void setDebugging(boolean debuggingEnabled) {
 +    Logger.getLogger(Constants.CORE_PACKAGE_NAME).setLevel(debuggingEnabled ? Level.TRACE : Level.INFO);
 +  }
 +
 +  public static final boolean isDebuggingEnabled() {
 +    return Logger.getLogger(Constants.CORE_PACKAGE_NAME).isTraceEnabled();
 +  }
 +
 +  private final void printHelp(String usage, String description, Options opts) {
 +    printHelp(usage, description, opts, Integer.MAX_VALUE);
 +  }
 +
 +  private final void printHelp(String usage, String description, Options opts, int width) {
 +    // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
 +    new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
 +    writer.flush();
 +  }
 +
 +  public int getExitCode() {
 +    return exitCode;
 +  }
 +
 +  public void resetExitCode() {
 +    exitCode = 0;
 +  }
 +
 +  public void setExit(boolean exit) {
 +    this.exit = exit;
 +  }
 +
 +  public boolean getExit() {
 +    return this.exit;
 +  }
 +
 +  public boolean isVerbose() {
 +    return verbose;
 +  }
 +
 +  public void setTableName(String tableName) {
 +    this.tableName = (tableName == null || tableName.isEmpty()) ? "" : Tables.qualified(tableName);
 +  }
 +
 +  public String getTableName() {
 +    return tableName;
 +  }
 +
 +  public ConsoleReader getReader() {
 +    return reader;
 +  }
 +
 +  public void updateUser(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
 +    connector = instance.getConnector(principal, token);
 +    this.principal = principal;
 +    this.token = token;
 +  }
 +
 +  public String getPrincipal() {
 +    return principal;
 +  }
 +
 +  public AuthenticationToken getToken() {
 +    return token;
 +  }
 +
 +  /**
 +   * Return the formatter for the current table.
 +   * 
 +   * @return the formatter class for the current table
 +   */
 +  public Class<? extends Formatter> getFormatter() {
 +    return getFormatter(this.tableName);
 +  }
 +
 +  /**
 +   * Return the formatter for the given table.
 +   * 
 +   * @param tableName
 +   *          the table name
 +   * @return the formatter class for the given table
 +   */
 +  public Class<? extends Formatter> getFormatter(String tableName) {
 +    Class<? extends Formatter> formatter = FormatterCommand.getCurrentFormatter(tableName, this);
 +
 +    if (null == formatter) {
 +      logError("Could not load the specified formatter. Using the DefaultFormatter");
 +      return this.defaultFormatterClass;
 +    } else {
 +      return formatter;
 +    }
 +  }
 +
 +  public void setLogErrorsToConsole() {
 +    this.logErrorsToConsole = true;
 +  }
 +
 +  private void logError(String s) {
 +    log.error(s);
 +    if (logErrorsToConsole) {
 +      try {
 +        reader.println("ERROR: " + s);
 +        reader.flush();
 +      } catch (IOException e) {}
 +    }
 +  }
 +
 +  public String readMaskedLine(String prompt, Character mask) throws IOException {
 +    this.masking = true;
 +    String s = reader.readLine(prompt, mask);
 +    this.masking = false;
 +    return s;
 +  }
 +
 +  public boolean isMasking() {
 +    return masking;
 +  }
 +
 +  public boolean hasExited() {
 +    return exit;
 +  }
 +
 +  public boolean isTabCompletion() {
 +    return tabCompletion;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
index d8b9f65,0000000..966fb00
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
@@@ -1,60 -1,0 +1,61 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileNotFoundException;
- import java.nio.charset.StandardCharsets;
 +import java.util.List;
 +import java.util.Scanner;
 +
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.hadoop.io.Text;
 +
 +import com.google.common.collect.Lists;
 +
 +public class ShellUtil {
 +
 +  /**
 +   * Scans the given file line-by-line (ignoring empty lines) and returns a list containing those lines. If decode is set to true, every line is decoded using
 +   * {@link Base64#decodeBase64(byte[])} from the UTF-8 bytes of that line before inserting in the list.
 +   * 
 +   * @param filename
 +   *          Path to the file that needs to be scanned
 +   * @param decode
 +   *          Whether to decode lines in the file
 +   * @return List of {@link Text} objects containing data in the given file
 +   * @throws FileNotFoundException
 +   *           if the given file doesn't exist
 +   */
 +  public static List<Text> scanFile(String filename, boolean decode) throws FileNotFoundException {
 +    String line;
-     Scanner file = new Scanner(new File(filename), StandardCharsets.UTF_8.name());
++    Scanner file = new Scanner(new File(filename), UTF_8.name());
 +    List<Text> result = Lists.newArrayList();
 +    try {
 +      while (file.hasNextLine()) {
 +        line = file.nextLine();
 +        if (!line.isEmpty()) {
-           result.add(decode ? new Text(Base64.decodeBase64(line.getBytes(StandardCharsets.UTF_8))) : new Text(line));
++          result.add(decode ? new Text(Base64.decodeBase64(line.getBytes(UTF_8))) : new Text(line));
 +        }
 +      }
 +    } finally {
 +      file.close();
 +    }
 +    return result;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
index 807de24,0000000..ef35f52
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
@@@ -1,66 -1,0 +1,67 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.commands;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 +import java.util.Map;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.Token;
 +import org.apache.accumulo.shell.Shell.Command;
 +import org.apache.commons.cli.CommandLine;
 +
 +public class AuthenticateCommand extends Command {
 +  @Override
 +  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, IOException {
 +    final String user = cl.getArgs()[0];
 +    final String p = shellState.readMaskedLine("Enter current password for '" + user + "': ", '*');
 +    if (p == null) {
 +      shellState.getReader().println();
 +      return 0;
 +    } // user canceled
-     final byte[] password = p.getBytes(StandardCharsets.UTF_8);
++    final byte[] password = p.getBytes(UTF_8);
 +    final boolean valid = shellState.getConnector().securityOperations().authenticateUser(user, new PasswordToken(password));
 +    shellState.getReader().println((valid ? "V" : "Not v") + "alid");
 +    return 0;
 +  }
 +  
 +  @Override
 +  public String description() {
 +    return "verifies a user's credentials";
 +  }
 +  
 +  @Override
 +  public String usage() {
 +    return getName() + " <username>";
 +  }
 +  
 +  @Override
 +  public void registerCompletion(final Token root, final Map<Command.CompletionSet,Set<String>> completionSet) {
 +    registerCompletionForUsers(root, completionSet);
 +  }
 +  
 +  @Override
 +  public int numArgs() {
 +    return 1;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
index b384d5c,0000000..fb8af7f
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
@@@ -1,67 -1,0 +1,68 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.commands;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.File;
- import java.nio.charset.StandardCharsets;
 +import java.util.Scanner;
 +
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.Shell.Command;
 +import org.apache.commons.cli.CommandLine;
 +import org.apache.commons.cli.Option;
 +import org.apache.commons.cli.Options;
 +
 +public class ExecfileCommand extends Command {
 +  private Option verboseOption;
 +  
 +  @Override
 +  public String description() {
 +    return "specifies a file containing accumulo commands to execute";
 +  }
 +  
 +  @Override
 +  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
-     Scanner scanner = new Scanner(new File(cl.getArgs()[0]), StandardCharsets.UTF_8.name());
++    Scanner scanner = new Scanner(new File(cl.getArgs()[0]), UTF_8.name());
 +    try {
 +      while (scanner.hasNextLine()) {
 +        shellState.execCommand(scanner.nextLine(), true, cl.hasOption(verboseOption.getOpt()));
 +      }
 +    } finally {
 +      scanner.close();
 +    }
 +    return 0;
 +  }
 +  
 +  @Override
 +  public String usage() {
 +    return getName() + " <fileName>";
 +  }
 +  
 +  @Override
 +  public int numArgs() {
 +    return 1;
 +  }
 +  
 +  @Override
 +  public Options getOptions() {
 +    final Options opts = new Options();
 +    verboseOption = new Option("v", "verbose", false, "display command prompt as commands are executed");
 +    opts.addOption(verboseOption);
 +    return opts;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
index 8cbf214,0000000..f21e33a
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
@@@ -1,62 -1,0 +1,63 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.commands;
 +
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.security.SecureRandom;
 +import java.util.Random;
 +
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.ShellCommandException;
 +import org.apache.accumulo.shell.Shell.Command;
 +import org.apache.accumulo.shell.ShellCommandException.ErrorCode;
 +import org.apache.commons.cli.CommandLine;
 +
 +public class HiddenCommand extends Command {
 +  private static Random rand = new SecureRandom();
 +  
 +  @Override
 +  public String description() {
 +    return "The first rule of Accumulo is: \"You don't talk about Accumulo.\"";
 +  }
 +  
 +  @Override
 +  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
 +    if (rand.nextInt(10) == 0) {
 +      shellState.getReader().beep();
 +      shellState.getReader().println();
 +      shellState.getReader().println(
 +          new String(Base64.decodeBase64(("ICAgICAgIC4tLS4KICAgICAgLyAvXCBcCiAgICAgKCAvLS1cICkKICAgICAuPl8gIF88LgogICAgLyB8ICd8ICcgXAog"
 +              + "ICAvICB8Xy58Xy4gIFwKICAvIC98ICAgICAgfFwgXAogfCB8IHwgfFwvfCB8IHwgfAogfF98IHwgfCAgfCB8IHxffAogICAgIC8gIF9fICBcCiAgICAvICAv"
-               + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8));
++              + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(UTF_8)), UTF_8));
 +    } else {
 +      throw new ShellCommandException(ErrorCode.UNRECOGNIZED_COMMAND, getName());
 +    }
 +    return 0;
 +  }
 +  
 +  @Override
 +  public int numArgs() {
 +    return Shell.NO_FIXED_ARG_LENGTH_CHECK;
 +  }
 +  
 +  @Override
 +  public String getName() {
 +    return "accvmvlo";
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
index 86c78fc,0000000..dd38889
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
@@@ -1,97 -1,0 +1,97 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.commands;
 +
- import java.io.IOException;
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
 +
++import java.io.IOException;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.Shell.Command;
 +import org.apache.accumulo.shell.ShellOptions;
 +import org.apache.commons.cli.CommandLine;
 +import org.apache.commons.cli.Option;
 +import org.apache.commons.cli.Options;
 +
 +public class PasswdCommand extends Command {
 +  private Option userOpt;
 +  
 +  @Override
 +  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, IOException {
 +    final String currentUser = shellState.getConnector().whoami();
 +    final String user = cl.getOptionValue(userOpt.getOpt(), currentUser);
 +    
 +    String password = null;
 +    String passwordConfirm = null;
 +    String oldPassword = null;
 +    
 +    oldPassword = shellState.readMaskedLine("Enter current password for '" + currentUser + "': ", '*');
 +    if (oldPassword == null) {
 +      shellState.getReader().println();
 +      return 0;
 +    } // user canceled
 +    
 +    if (!shellState.getConnector().securityOperations().authenticateUser(currentUser, new PasswordToken(oldPassword)))
 +      throw new AccumuloSecurityException(user, SecurityErrorCode.BAD_CREDENTIALS);
 +    
 +    password = shellState.readMaskedLine("Enter new password for '" + user + "': ", '*');
 +    if (password == null) {
 +      shellState.getReader().println();
 +      return 0;
 +    } // user canceled
 +    passwordConfirm = shellState.readMaskedLine("Please confirm new password for '" + user + "': ", '*');
 +    if (passwordConfirm == null) {
 +      shellState.getReader().println();
 +      return 0;
 +    } // user canceled
 +    
 +    if (!password.equals(passwordConfirm)) {
 +      throw new IllegalArgumentException("Passwords do not match");
 +    }
-     byte[] pass = password.getBytes(StandardCharsets.UTF_8);
++    byte[] pass = password.getBytes(UTF_8);
 +    shellState.getConnector().securityOperations().changeLocalUserPassword(user, new PasswordToken(pass));
 +    // update the current credentials if the password changed was for
 +    // the current user
 +    if (shellState.getConnector().whoami().equals(user)) {
 +      shellState.updateUser(user, new PasswordToken(pass));
 +    }
 +    Shell.log.debug("Changed password for user " + user);
 +    return 0;
 +  }
 +  
 +  @Override
 +  public String description() {
 +    return "changes a user's password";
 +  }
 +  
 +  @Override
 +  public Options getOptions() {
 +    final Options o = new Options();
 +    userOpt = new Option(ShellOptions.userOption, "user", true, "user to operate on");
 +    userOpt.setArgName("user");
 +    o.addOption(userOpt);
 +    return o;
 +  }
 +  
 +  @Override
 +  public int numArgs() {
 +    return 0;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
index ecc51eb,0000000..2fdb7fe
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
@@@ -1,141 -1,0 +1,142 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.commands;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.UnsupportedEncodingException;
- import java.nio.charset.StandardCharsets;
 +import java.util.ArrayList;
 +import java.util.Iterator;
 +
 +import org.apache.accumulo.core.util.BadArgumentException;
 +import org.apache.accumulo.shell.Shell;
 +
 +/**
 + * A basic tokenizer for generating tokens from a string. It understands quoted strings and escaped quote characters.
 + * 
 + * You can use the escape sequence '\' to escape single quotes, double quotes, and spaces only, in addition to the escape character itself.
 + * 
 + * The behavior is the same for single and double quoted strings. (i.e. '\'' is the same as "\'")
 + */
 +
 +public class QuotedStringTokenizer implements Iterable<String> {
 +  private ArrayList<String> tokens;
 +  private String input;
 +  
 +  public QuotedStringTokenizer(final String t) throws BadArgumentException {
 +    tokens = new ArrayList<String>();
 +    this.input = t;
 +    try {
 +      createTokens();
 +    } catch (UnsupportedEncodingException e) {
 +      throw new IllegalArgumentException(e.getMessage());
 +    }
 +  }
 +  
 +  public String[] getTokens() {
 +    return tokens.toArray(new String[tokens.size()]);
 +  }
 +  
 +  private void createTokens() throws BadArgumentException, UnsupportedEncodingException {
 +    boolean inQuote = false;
 +    boolean inEscapeSequence = false;
 +    String hexChars = null;
 +    char inQuoteChar = '"';
 +    
 +    final byte[] token = new byte[input.length()];
 +    int tokenLength = 0;
-     final byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
++    final byte[] inputBytes = input.getBytes(UTF_8);
 +    for (int i = 0; i < input.length(); ++i) {
 +      final char ch = input.charAt(i);
 +      
 +      // if I ended up in an escape sequence, check for valid escapable character, and add it as a literal
 +      if (inEscapeSequence) {
 +        inEscapeSequence = false;
 +        if (ch == 'x') {
 +          hexChars = "";
 +        } else if (ch == ' ' || ch == '\'' || ch == '"' || ch == '\\') {
 +          token[tokenLength++] = inputBytes[i];
 +        } else {
 +          throw new BadArgumentException("can only escape single quotes, double quotes, the space character, the backslash, and hex input", input, i);
 +        }
 +      }
 +      // in a hex escape sequence
 +      else if (hexChars != null) {
 +        final int digit = Character.digit(ch, 16);
 +        if (digit < 0) {
 +          throw new BadArgumentException("expected hex character", input, i);
 +        }
 +        hexChars += ch;
 +        if (hexChars.length() == 2) {
 +          byte b;
 +          try {
 +            b = (byte) (0xff & Short.parseShort(hexChars, 16));
 +            if (!Character.isValidCodePoint(0xff & b))
 +              throw new NumberFormatException();
 +          } catch (NumberFormatException e) {
 +            throw new BadArgumentException("unsupported non-ascii character", input, i);
 +          }
 +          token[tokenLength++] = b;
 +          hexChars = null;
 +        }
 +      }
 +      // in a quote, either end the quote, start escape, or continue a token
 +      else if (inQuote) {
 +        if (ch == inQuoteChar) {
 +          inQuote = false;
 +          tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
 +          tokenLength = 0;
 +        } else if (ch == '\\') {
 +          inEscapeSequence = true;
 +        } else {
 +          token[tokenLength++] = inputBytes[i];
 +        }
 +      }
 +      // not in a quote, either enter a quote, end a token, start escape, or continue a token
 +      else {
 +        if (ch == '\'' || ch == '"') {
 +          if (tokenLength > 0) {
 +            tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
 +            tokenLength = 0;
 +          }
 +          inQuote = true;
 +          inQuoteChar = ch;
 +        } else if (ch == ' ' && tokenLength > 0) {
 +          tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
 +          tokenLength = 0;
 +        } else if (ch == '\\') {
 +          inEscapeSequence = true;
 +        } else if (ch != ' ') {
 +          token[tokenLength++] = inputBytes[i];
 +        }
 +      }
 +    }
 +    if (inQuote) {
 +      throw new BadArgumentException("missing terminating quote", input, input.length());
 +    } else if (inEscapeSequence || hexChars != null) {
 +      throw new BadArgumentException("escape sequence not complete", input, input.length());
 +    }
 +    if (tokenLength > 0) {
 +      tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
 +    }
 +  }
 +  
 +  @Override
 +  public Iterator<String> iterator() {
 +    return tokens.iterator();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
index 491f144,0000000..6e7247e
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
@@@ -1,71 -1,0 +1,72 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.commands;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 +import java.util.Map;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.Token;
 +import org.apache.accumulo.shell.Shell.Command;
 +import org.apache.commons.cli.CommandLine;
 +
 +public class UserCommand extends Command {
 +  @Override
 +  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, IOException {
 +    // save old credentials and connection in case of failure
 +    String user = cl.getArgs()[0];
 +    byte[] pass;
 +    
 +    // We can't let the wrapping try around the execute method deal
 +    // with the exceptions because we have to do something if one
 +    // of these methods fails
 +    final String p = shellState.readMaskedLine("Enter password for user " + user + ": ", '*');
 +    if (p == null) {
 +      shellState.getReader().println();
 +      return 0;
 +    } // user canceled
-     pass = p.getBytes(StandardCharsets.UTF_8);
++    pass = p.getBytes(UTF_8);
 +    shellState.updateUser(user, new PasswordToken(pass));
 +    return 0;
 +  }
 +  
 +  @Override
 +  public String description() {
 +    return "switches to the specified user";
 +  }
 +  
 +  @Override
 +  public void registerCompletion(final Token root, final Map<Command.CompletionSet,Set<String>> special) {
 +    registerCompletionForUsers(root, special);
 +  }
 +  
 +  @Override
 +  public String usage() {
 +    return getName() + " <username>";
 +  }
 +  
 +  @Override
 +  public int numArgs() {
 +    return 1;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
index cf028eb,0000000..f862b68
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
@@@ -1,143 -1,0 +1,143 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.shell.mock;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.OutputStream;
- import java.nio.charset.StandardCharsets;
- 
 +import jline.console.ConsoleReader;
 +
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.shell.Shell;
 +import org.apache.accumulo.shell.ShellOptionsJC;
 +
 +/**
 + * An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes.
 + */
 +public class MockShell extends Shell {
 +  private static final String NEWLINE = "\n";
 +  
 +  protected InputStream in;
 +  protected OutputStream out;
 +
 +  public MockShell(InputStream in, OutputStream out) throws IOException {
 +    super();
 +    this.in = in;
 +    this.out = out;
 +  }
 +
 +  public boolean config(String... args) {
 +    configError = super.config(args);
 +    
 +    // Update the ConsoleReader with the input and output "redirected"
 +    try {
 +      this.reader = new ConsoleReader(in, out);
 +    } catch (Exception e) {
 +      printException(e);
 +      configError = true;
 +    }
 +    
 +    // Don't need this for testing purposes
 +    this.reader.setHistoryEnabled(false);
 +    this.reader.setPaginationEnabled(false);
 +    
 +    // Make the parsing from the client easier;
 +    this.verbose = false;
 +    return configError;
 +  }
 +  
 +  @Override
 +  protected void setInstance(ShellOptionsJC options) {
 +    // We always want a MockInstance for this test
 +    instance = new MockInstance();
 +  }
 +
 +  public int start() throws IOException {
 +    if (configError)
 +      return 1;
 +    
 +    String input;
 +    if (isVerbose())
 +      printInfo();
 +    
 +    if (execFile != null) {
-       java.util.Scanner scanner = new java.util.Scanner(execFile, StandardCharsets.UTF_8.name());
++      java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
 +      try {
 +        while (scanner.hasNextLine() && !hasExited()) {
 +          execCommand(scanner.nextLine(), true, isVerbose());
 +        }
 +      } finally {
 +        scanner.close();
 +      }
 +    } else if (execCommand != null) {
 +      for (String command : execCommand.split("\n")) {
 +        execCommand(command, true, isVerbose());
 +      }
 +      return exitCode;
 +    }
 +    
 +    while (true) {
 +      if (hasExited())
 +        return exitCode;
 +      
 +      reader.setPrompt(getDefaultPrompt());
 +      input = reader.readLine();
 +      if (input == null) {
 +        reader.println();
 +        return exitCode;
 +      } // user canceled
 +      
 +      execCommand(input, false, false);
 +    }
 +  }
 +  
 +  /**
 +   * @param in
 +   *          the in to set
 +   */
 +  public void setConsoleInputStream(InputStream in) {
 +    this.in = in;
 +  }
 +  
 +  /**
 +   * @param out
 +   *          the output stream to set
 +   */
 +  public void setConsoleWriter(OutputStream out) {
 +    this.out = out;
 +  }
 +
 +  /**
 +   * Convenience method to create the byte-array to hand to the console
 +   * 
 +   * @param commands
 +   *          An array of commands to run
 +   * @return A byte[] input stream which can be handed to the console.
 +   */
 +  public static ByteArrayInputStream makeCommands(String... commands) {
 +    StringBuilder sb = new StringBuilder(commands.length * 8);
 +    
 +    for (String command : commands) {
 +      sb.append(command).append(NEWLINE);
 +    }
 +    
-     return new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
++    return new ByteArrayInputStream(sb.toString().getBytes(UTF_8));
 +  }
 +}


[07/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index 665b132,0000000..4975428
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -1,356 -1,0 +1,357 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.OutputStream;
 +import java.io.OutputStreamWriter;
 +import java.io.PrintWriter;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.UUID;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +import javax.servlet.http.HttpSession;
 +
 +import jline.console.ConsoleReader;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.shell.Shell;
 +
 +public class ShellServlet extends BasicServlet {
 +  private static final long serialVersionUID = 1L;
 +  private Map<String,ShellExecutionThread> userShells = new HashMap<String,ShellExecutionThread>();
 +  private ExecutorService service = Executors.newCachedThreadPool();
 +
 +  public static final String CSRF_KEY = "csrf_token";
 +
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Shell";
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws IOException {
 +    HttpSession session = req.getSession(true);
 +    final String CSRF_TOKEN;
 +    if (null == session.getAttribute(CSRF_KEY)) {
 +      // No token, make one
 +      CSRF_TOKEN = UUID.randomUUID().toString();
 +      session.setAttribute(CSRF_KEY, CSRF_TOKEN);
 +    } else {
 +      // Pull the token out of the session
 +      CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
 +      if (null == CSRF_TOKEN) {
 +        throw new RuntimeException("No valid CSRF token exists in session");
 +      }
 +    }
 +
 +    String user = (String) session.getAttribute("user");
 +    if (user == null) {
 +      // user attribute is null, check to see if username and password are passed as parameters
 +      user = req.getParameter("user");
 +      String pass = req.getParameter("pass");
 +      String mock = req.getParameter("mock");
 +      if (user == null || pass == null) {
 +        // username or password are null, re-authenticate
 +        sb.append(authenticationForm(req.getRequestURI(), CSRF_TOKEN));
 +        return;
 +      }
 +      try {
 +        // get a new shell for this user
 +        ShellExecutionThread shellThread = new ShellExecutionThread(user, pass, mock);
 +        service.submit(shellThread);
 +        userShells.put(session.getId(), shellThread);
 +      } catch (IOException e) {
 +        // error validating user, reauthenticate
 +        sb.append("<div id='loginError'>Invalid user/password</div>" + authenticationForm(req.getRequestURI(), CSRF_TOKEN));
 +        return;
 +      }
 +      session.setAttribute("user", user);
 +    }
 +    if (!userShells.containsKey(session.getId())) {
 +      // no existing shell for this user, re-authenticate
 +      sb.append(authenticationForm(req.getRequestURI(), UUID.randomUUID().toString()));
 +      return;
 +    }
 +
 +    ShellExecutionThread shellThread = userShells.get(session.getId());
 +    shellThread.getOutput();
 +    shellThread.printInfo();
 +    sb.append("<div id='shell'>\n");
 +    sb.append("<pre id='shellResponse'>").append(shellThread.getOutput()).append("</pre>\n");
 +    sb.append("<form><span id='shellPrompt'>").append(shellThread.getPrompt());
 +    sb.append("</span><input type='text' name='cmd' id='cmd' onkeydown='return handleKeyDown(event.keyCode);'>\n");
 +    sb.append("</form>\n</div>\n");
 +    sb.append("<script type='text/javascript'>\n");
 +    sb.append("var url = '").append(req.getRequestURL().toString()).append("';\n");
 +    sb.append("var xmlhttp = new XMLHttpRequest();\n");
 +    sb.append("var hsize = 1000;\n");
 +    sb.append("var hindex = 0;\n");
 +    sb.append("var history = new Array();\n");
 +    sb.append("\n");
 +    sb.append("function handleKeyDown(keyCode) {\n");
 +    sb.append("  if (keyCode==13) {\n");
 +    sb.append("    submitCmd(document.getElementById('cmd').value);\n");
 +    sb.append("    hindex = history.length;\n");
 +    sb.append("    return false;\n");
 +    sb.append("  } else if (keyCode==38) {\n");
 +    sb.append("    hindex = hindex==0 ? history.length : hindex - 1;\n");
 +    sb.append("    if (hindex == history.length)\n");
 +    sb.append("      document.getElementById('cmd').value = '';\n");
 +    sb.append("    else\n");
 +    sb.append("      document.getElementById('cmd').value = history[hindex];\n");
 +    sb.append("    return false;\n");
 +    sb.append("  } else if (keyCode==40) {\n");
 +    sb.append("    hindex = hindex==history.length ? history.length : hindex + 1;\n");
 +    sb.append("    if (hindex == history.length)\n");
 +    sb.append("      document.getElementById('cmd').value = '';\n");
 +    sb.append("    else\n");
 +    sb.append("      document.getElementById('cmd').value = history[hindex];\n");
 +    sb.append("    return false;\n");
 +    sb.append("  }\n");
 +    sb.append("  return true;\n");
 +    sb.append("}\n");
 +    sb.append("\n");
 +    sb.append("function submitCmd(cmd) {\n");
 +    sb.append("  if (cmd=='history') {\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += document.getElementById('shellPrompt').innerHTML+cmd+'\\n';\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += history.join('\\n');\n");
 +    sb.append("    return\n");
 +    sb.append("  }\n");
 +    sb.append("  xmlhttp.open('POST',url+'?cmd='+cmd+'&'+'").append(CSRF_KEY).append("=").append(CSRF_TOKEN).append("',false);\n");
 +    sb.append("  xmlhttp.send();\n");
 +    sb.append("  var text = xmlhttp.responseText;\n");
 +    sb.append("  var index = text.lastIndexOf('\\n');\n");
 +    sb.append("  if (index >= 0) {\n");
 +    sb.append("    if (index > 0 && document.getElementById('cmd').type == 'text') {\n");
 +    sb.append("      if (history.length == hsize)\n");
 +    sb.append("        history.shift()\n");
 +    sb.append("      history.push(cmd)\n");
 +    sb.append("    }\n");
 +    sb.append("    if (text.charAt(text.length-1)=='*') {\n");
 +    sb.append("      document.getElementById('cmd').type = 'password';\n");
 +    sb.append("      text = text.substring(0,xmlhttp.responseText.length-2);\n");
 +    sb.append("    } else {\n");
 +    sb.append("      document.getElementById('cmd').type = 'text';\n");
 +    sb.append("    }\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += text.substring(0,index+1);\n");
 +    sb.append("    document.getElementById('shellPrompt').innerHTML = text.substring(index+1);\n");
 +    sb.append("    document.getElementById('cmd').value = '';\n");
 +    sb.append("    document.getElementById('shell').scrollTop = document.getElementById('cmd').offsetTop;\n");
 +    sb.append("  } else {\n");
 +    sb.append("    window.location = url;\n");
 +    sb.append("  }\n");
 +    sb.append("}\n");
 +    sb.append("</script>\n");
 +    sb.append("<script type='text/javascript'>window.onload = function() { document.getElementById('cmd').select(); }</script>\n");
 +  }
 +
 +  @Override
 +  protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    final HttpSession session = req.getSession(true);
 +    String user = (String) session.getAttribute("user");
 +    if (user == null || !userShells.containsKey(session.getId())) {
 +      // no existing shell for user, re-authenticate
 +      doGet(req, resp);
 +      return;
 +    }
 +    final String CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
 +    if (null == CSRF_TOKEN) {
 +      // no csrf token, need to re-auth
 +      doGet(req, resp);
 +    }
 +    ShellExecutionThread shellThread = userShells.get(session.getId());
 +    String cmd = req.getParameter("cmd");
 +    if (cmd == null) {
 +      // the command is null, just print prompt
 +      resp.getWriter().append(shellThread.getPrompt());
 +      resp.getWriter().flush();
 +      return;
 +    }
 +    shellThread.addInputString(cmd);
 +    shellThread.waitUntilReady();
 +    if (shellThread.isDone()) {
 +      // the command was exit, invalidate session
 +      userShells.remove(session.getId());
 +      session.invalidate();
 +      return;
 +    }
 +    // get the shell's output
 +    StringBuilder sb = new StringBuilder();
 +    sb.append(shellThread.getOutput().replace("<", "&lt;").replace(">", "&gt;"));
 +    if (sb.length() == 0 || !(sb.charAt(sb.length() - 1) == '\n'))
 +      sb.append("\n");
 +    // check if shell is waiting for input
 +    if (!shellThread.isWaitingForInput())
 +      sb.append(shellThread.getPrompt());
 +    // check if shell is waiting for password input
 +    if (shellThread.isMasking())
 +      sb.append("*");
 +    resp.getWriter().append(sb.toString());
 +    resp.getWriter().flush();
 +  }
 +
 +  private String authenticationForm(String requestURI, String csrfToken) {
 +    return "<div id='login'><form method=POST action='" + requestURI + "'>"
 +        + "<table><tr><td>Mock:&nbsp</td><td><input type='checkbox' name='mock' value='mock'></td></tr>"
 +        + "<tr><td>Username:&nbsp;</td><td><input type='text' name='user'></td></tr>"
 +        + "<tr><td>Password:&nbsp;</td><td><input type='password' name='pass'></td><td>"
 +        + "<input type='hidden' name='" + CSRF_KEY + "' value='" + csrfToken + "'/><input type='submit' value='Enter'></td></tr></table></form></div>";
 +  }
 +  
 +  private static class StringBuilderOutputStream extends OutputStream {
 +    StringBuilder sb = new StringBuilder();
 +    
 +    @Override
 +    public void write(int b) throws IOException {
 +      sb.append((char) (0xff & b));
 +    }
 +    
 +    public String get() {
 +      return sb.toString();
 +    }
 +    
 +    public void clear() {
 +      sb.setLength(0);
 +    }
 +  }
 +  
 +  private static class ShellExecutionThread extends InputStream implements Runnable {
 +    private Shell shell;
 +    StringBuilderOutputStream output;
 +    private String cmd;
 +    private int cmdIndex;
 +    private boolean done;
 +    private boolean readWait;
 +    
 +    private ShellExecutionThread(String username, String password, String mock) throws IOException {
 +      this.done = false;
 +      this.cmd = null;
 +      this.cmdIndex = 0;
 +      this.readWait = false;
 +      this.output = new StringBuilderOutputStream();
 +      ConsoleReader reader = new ConsoleReader(this, output);
-       this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, Constants.UTF8)));
++      this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, UTF_8)));
 +      shell.setLogErrorsToConsole();
 +      if (mock != null) {
 +        if (shell.config("--fake", "-u", username, "-p", password))
 +          throw new IOException("mock shell config error");
 +      } else if (shell.config("-u", username, "-p", password)) {
 +        throw new IOException("shell config error");
 +      }
 +    }
 +    
 +    @Override
 +    public synchronized int read() throws IOException {
 +      if (cmd == null) {
 +        readWait = true;
 +        this.notifyAll();
 +      }
 +      while (cmd == null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {}
 +      }
 +      readWait = false;
 +      int c;
 +      if (cmdIndex == cmd.length())
 +        c = '\n';
 +      else
 +        c = cmd.charAt(cmdIndex);
 +      cmdIndex++;
 +      if (cmdIndex > cmd.length()) {
 +        cmd = null;
 +        cmdIndex = 0;
 +        this.notifyAll();
 +      }
 +      return c;
 +    }
 +    
 +    @Override
 +    public synchronized void run() {
 +      Thread.currentThread().setName("shell thread");
 +      while (!shell.hasExited()) {
 +        while (cmd == null) {
 +          try {
 +            this.wait();
 +          } catch (InterruptedException e) {}
 +        }
 +        String tcmd = cmd;
 +        cmd = null;
 +        cmdIndex = 0;
 +        try {
 +          shell.execCommand(tcmd, false, true);
 +        } catch (IOException e) {}
 +        this.notifyAll();
 +      }
 +      done = true;
 +      this.notifyAll();
 +    }
 +    
 +    public synchronized void addInputString(String s) {
 +      if (done)
 +        throw new IllegalStateException("adding string to exited shell");
 +      if (cmd == null) {
 +        cmd = s;
 +      } else {
 +        throw new IllegalStateException("adding string to shell not waiting for input");
 +      }
 +      this.notifyAll();
 +    }
 +    
 +    public synchronized void waitUntilReady() {
 +      while (cmd != null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +    
 +    public synchronized String getOutput() {
 +      String s = output.get();
 +      output.clear();
 +      return s;
 +    }
 +    
 +    public String getPrompt() {
 +      return shell.getDefaultPrompt();
 +    }
 +    
 +    public void printInfo() throws IOException {
 +      shell.printInfo();
 +    }
 +    
 +    public boolean isMasking() {
 +      return shell.isMasking();
 +    }
 +    
 +    public synchronized boolean isWaitingForInput() {
 +      return readWait;
 +    }
 +    
 +    public boolean isDone() {
 +      return done;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
index 0cbf957,0000000..3cd635e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
@@@ -1,179 -1,0 +1,180 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Map.Entry;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.master.thrift.Compacting;
 +import org.apache.accumulo.core.master.thrift.DeadServer;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.master.state.TabletServerState;
 +import org.apache.accumulo.server.util.TableInfoUtil;
 +
 +public class XMLServlet extends BasicServlet {
 +  private static final long serialVersionUID = 1L;
 +  
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "XML Report";
 +  }
 +  
 +  @Override
 +  protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
-     resp.setContentType("text/xml;charset=" + Constants.UTF8.name());
-     sb.append("<?xml version=\"1.0\" encoding=\"" + Constants.UTF8.name() + "\"?>\n");
++    resp.setContentType("text/xml;charset=" + UTF_8.name());
++    sb.append("<?xml version=\"1.0\" encoding=\"" + UTF_8.name() + "\"?>\n");
 +    sb.append("<stats>\n");
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
 +    double totalIngest = 0.;
 +    double totalQuery = 0.;
 +    double disk = 0.0;
 +    long totalEntries = 0L;
 +    
 +    sb.append("\n<servers>\n");
 +    if (Monitor.getMmi() == null || Monitor.getMmi().tableMap == null) {
 +      sb.append("</servers>\n");
 +      return;
 +    }
 +    SortedMap<String,TableInfo> tableStats = new TreeMap<String,TableInfo>(Monitor.getMmi().tableMap);
 +    
 +    for (TabletServerStatus status : Monitor.getMmi().tServerInfo) {
 +      
 +      sb.append("\n<server id='").append(status.name).append("'>\n");
 +      sb.append("<hostname>").append(TServerLinkType.displayName(status.name)).append("</hostname>");
 +      sb.append("<lastContact>").append(System.currentTimeMillis() - status.lastContact).append("</lastContact>\n");
 +      sb.append("<osload>").append(status.osLoad).append("</osload>\n");
 +      
 +      TableInfo summary = TableInfoUtil.summarizeTableStats(status);
 +      sb.append("<compactions>\n");
 +      sb.append("<major>").append("<running>").append(summary.majors.running).append("</running>").append("<queued>").append(summary.majors.queued)
 +          .append("</queued>").append("</major>\n");
 +      sb.append("<minor>").append("<running>").append(summary.minors.running).append("</running>").append("<queued>").append(summary.minors.queued)
 +          .append("</queued>").append("</minor>\n");
 +      sb.append("</compactions>\n");
 +      
 +      sb.append("<tablets>").append(summary.tablets).append("</tablets>\n");
 +      
 +      sb.append("<ingest>").append(summary.ingestRate).append("</ingest>\n");
 +      sb.append("<query>").append(summary.queryRate).append("</query>\n");
 +      sb.append("<ingestMB>").append(summary.ingestByteRate / 1000000.0).append("</ingestMB>\n");
 +      sb.append("<queryMB>").append(summary.queryByteRate / 1000000.0).append("</queryMB>\n");
 +      sb.append("<scans>").append(summary.scans.running + summary.scans.queued).append("</scans>");
 +      sb.append("<scansessions>").append(Monitor.getLookupRate()).append("</scansessions>\n");
 +      sb.append("<holdtime>").append(status.holdTime).append("</holdtime>\n");
 +      totalIngest += summary.ingestRate;
 +      totalQuery += summary.queryRate;
 +      totalEntries += summary.recs;
 +      sb.append("</server>\n");
 +    }
 +    sb.append("\n</servers>\n");
 +    
 +    sb.append("\n<masterGoalState>" + Monitor.getMmi().goalState + "</masterGoalState>\n");
 +    sb.append("\n<masterState>" + Monitor.getMmi().state + "</masterState>\n");
 +    
 +    sb.append("\n<badTabletServers>\n");
 +    for (Entry<String,Byte> entry : Monitor.getMmi().badTServers.entrySet()) {
 +      sb.append(String.format("<badTabletServer id='%s' status='%s'/>\n", entry.getKey(), TabletServerState.getStateById(entry.getValue())));
 +    }
 +    sb.append("\n</badTabletServers>\n");
 +    
 +    sb.append("\n<tabletServersShuttingDown>\n");
 +    for (String server : Monitor.getMmi().serversShuttingDown) {
 +      sb.append(String.format("<server id='%s'/>\n", server));
 +    }
 +    sb.append("\n</tabletServersShuttingDown>\n");
 +    
 +    sb.append(String.format("\n<unassignedTablets>%d</unassignedTablets>\n", Monitor.getMmi().unassignedTablets));
 +    
 +    sb.append("\n<deadTabletServers>\n");
 +    for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
 +      sb.append(String.format("<deadTabletServer id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
 +    }
 +    sb.append("\n</deadTabletServers>\n");
 +    
 +    sb.append("\n<deadLoggers>\n");
 +    for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
 +      sb.append(String.format("<deadLogger id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
 +    }
 +    sb.append("\n</deadLoggers>\n");
 +    
 +    sb.append("\n<tables>\n");
 +    Instance instance = HdfsZooInstance.getInstance();
 +    for (Entry<String,TableInfo> entry : tableStats.entrySet()) {
 +      TableInfo tableInfo = entry.getValue();
 +      
 +      sb.append("\n<table>\n");
 +      String tableId = entry.getKey();
 +      String tableName = "unknown";
 +      String tableState = "unknown";
 +      try {
 +        tableName = Tables.getTableName(instance, tableId);
 +        tableState = Tables.getTableState(instance, tableId).toString();
 +      } catch (Exception ex) {
 +        log.warn(ex, ex);
 +      }
 +      sb.append("<tablename>").append(tableName).append("</tablename>\n");
 +      sb.append("<tableId>").append(tableId).append("</tableId>\n");
 +      sb.append("<tableState>").append(tableState).append("</tableState>\n");
 +      sb.append("<tablets>").append(tableInfo.tablets).append("</tablets>\n");
 +      sb.append("<onlineTablets>").append(tableInfo.onlineTablets).append("</onlineTablets>\n");
 +      sb.append("<recs>").append(tableInfo.recs).append("</recs>\n");
 +      sb.append("<recsInMemory>").append(tableInfo.recsInMemory).append("</recsInMemory>\n");
 +      sb.append("<ingest>").append(tableInfo.ingestRate).append("</ingest>\n");
 +      sb.append("<ingestByteRate>").append(tableInfo.ingestByteRate).append("</ingestByteRate>\n");
 +      sb.append("<query>").append(tableInfo.queryRate).append("</query>\n");
 +      sb.append("<queryByteRate>").append(tableInfo.queryRate).append("</queryByteRate>\n");
 +      int running = 0;
 +      int queued = 0;
 +      Compacting compacting = entry.getValue().majors;
 +      if (compacting != null) {
 +        running = compacting.running;
 +        queued = compacting.queued;
 +      }
 +      sb.append("<majorCompactions>").append("<running>").append(running).append("</running>").append("<queued>").append(queued).append("</queued>")
 +          .append("</majorCompactions>\n");
 +      sb.append("</table>\n");
 +    }
 +    sb.append("\n</tables>\n");
 +    
 +    sb.append("\n<totals>\n");
 +    sb.append("<ingestrate>").append(totalIngest).append("</ingestrate>\n");
 +    sb.append("<queryrate>").append(totalQuery).append("</queryrate>\n");
 +    sb.append("<diskrate>").append(disk).append("</diskrate>\n");
 +    sb.append("<numentries>").append(totalEntries).append("</numentries>\n");
 +    sb.append("</totals>\n");
 +  }
 +  
 +  @Override
 +  protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
 +    sb.append("\n</stats>\n");
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 90bb0d4,0000000..1c464d4
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@@ -1,105 -1,0 +1,106 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets.trace;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Date;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +
 +import javax.servlet.http.HttpServletRequest;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.trace.TraceFormatter;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.servlets.BasicServlet;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +
 +abstract class Basic extends BasicServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  public static String getStringParameter(HttpServletRequest req, String name, String defaultValue) {
 +    String result = req.getParameter(name);
 +    if (result == null) {
 +      return defaultValue;
 +    }
 +    return result;
 +  }
 +
 +  public static int getIntParameter(HttpServletRequest req, String name, int defaultMinutes) {
 +    String valueString = req.getParameter(name);
 +    if (valueString == null)
 +      return defaultMinutes;
 +    int result = 0;
 +    try {
 +      result = Integer.parseInt(valueString);
 +    } catch (NumberFormatException ex) {
 +      return defaultMinutes;
 +    }
 +    return result;
 +  }
 +
 +  public static String dateString(long millis) {
 +    return TraceFormatter.formatDate(new Date(millis));
 +  }
 +
 +  protected Scanner getScanner(StringBuilder sb) throws AccumuloException, AccumuloSecurityException {
 +    AccumuloConfiguration conf = Monitor.getSystemConfiguration();
 +    String principal = conf.get(Property.TRACE_USER);
 +    AuthenticationToken at;
 +    Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
 +    if (loginMap.isEmpty()) {
 +      Property p = Property.TRACE_PASSWORD;
-       at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++      at = new PasswordToken(conf.get(p).getBytes(UTF_8));
 +    } else {
 +      Properties props = new Properties();
 +      int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
 +      for (Entry<String,String> entry : loginMap.entrySet()) {
 +        props.put(entry.getKey().substring(prefixLength), entry.getValue());
 +      }
 +
 +      AuthenticationToken token = Property.createInstanceFromPropertyName(conf, Property.TRACE_TOKEN_TYPE, AuthenticationToken.class, new PasswordToken());
 +      token.init(props);
 +      at = token;
 +    }
 +
 +    String table = conf.get(Property.TRACE_TABLE);
 +    try {
 +      Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
 +      if (!conn.tableOperations().exists(table)) {
 +        return new NullScanner();
 +      }
 +      Scanner scanner = conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
 +      return scanner;
 +    } catch (AccumuloSecurityException ex) {
 +      sb.append("<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
 +      return null;
 +    } catch (TableNotFoundException ex) {
 +      return new NullScanner();
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
----------------------------------------------------------------------
diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index e3ae0c8,0000000..6094911
mode 100644,000000..100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -1,324 -1,0 +1,326 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tracer;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.net.InetSocketAddress;
 +import java.net.ServerSocket;
 +import java.nio.channels.ServerSocketChannel;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.atomic.AtomicReference;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.user.AgeOffFilter;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.trace.TraceFormatter;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.thrift.RemoteSpan;
 +import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
 +import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TByteArrayOutputStream;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.protocol.TCompactProtocol;
 +import org.apache.thrift.server.TServer;
 +import org.apache.thrift.server.TThreadPoolServer;
 +import org.apache.thrift.transport.TServerSocket;
 +import org.apache.thrift.transport.TServerTransport;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.Watcher.Event.EventType;
 +import org.apache.zookeeper.Watcher.Event.KeeperState;
 +
 +public class TraceServer implements Watcher {
 +
 +  final private static Logger log = Logger.getLogger(TraceServer.class);
 +  final private ServerConfiguration serverConfiguration;
 +  final private TServer server;
 +  final private AtomicReference<BatchWriter> writer;
 +  final private Connector connector;
 +  final String table;
 +
 +  private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) {
 +    m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
 +  }
 +
 +  static class ByteArrayTransport extends TTransport {
 +    TByteArrayOutputStream out = new TByteArrayOutputStream();
 +
 +    @Override
 +    public boolean isOpen() {
 +      return true;
 +    }
 +
 +    @Override
 +    public void open() throws TTransportException {}
 +
 +    @Override
 +    public void close() {}
 +
 +    @Override
 +    public int read(byte[] buf, int off, int len) {
 +      return 0;
 +    }
 +
 +    @Override
 +    public void write(byte[] buf, int off, int len) throws TTransportException {
 +      out.write(buf, off, len);
 +    }
 +
 +    public byte[] get() {
 +      return out.get();
 +    }
 +
 +    public int len() {
 +      return out.len();
 +    }
 +  }
 +
 +  class Receiver implements Iface {
 +    @Override
 +    public void span(RemoteSpan s) throws TException {
 +      String idString = Long.toHexString(s.traceId);
 +      String startString = Long.toHexString(s.start);
 +      Mutation spanMutation = new Mutation(new Text(idString));
 +      Mutation indexMutation = new Mutation(new Text("idx:" + s.svc + ":" + startString));
 +      long diff = s.stop - s.start;
-       indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(Constants.UTF8)));
++      indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(UTF_8)));
 +      ByteArrayTransport transport = new ByteArrayTransport();
 +      TCompactProtocol protocol = new TCompactProtocol(transport);
 +      s.write(protocol);
 +      String parentString = Long.toHexString(s.parentId);
 +      if (s.parentId == Span.ROOT_SPAN_ID)
 +        parentString = "";
 +      put(spanMutation, "span", parentString + ":" + Long.toHexString(s.spanId), transport.get(), transport.len());
 +      // Map the root span to time so we can look up traces by time
 +      Mutation timeMutation = null;
 +      if (s.parentId == Span.ROOT_SPAN_ID) {
 +        timeMutation = new Mutation(new Text("start:" + startString));
 +        put(timeMutation, "id", idString, transport.get(), transport.len());
 +      }
 +      try {
 +        final BatchWriter writer = TraceServer.this.writer.get();
 +        /*
 +         * Check for null, because we expect spans to come in much faster than flush calls. In the case of failure, we'd rather avoid logging tons of NPEs.
 +         */
 +        if (null == writer) {
 +          log.warn("writer is not ready; discarding span.");
 +          return;
 +        }
 +        writer.addMutation(spanMutation);
 +        writer.addMutation(indexMutation);
 +        if (timeMutation != null)
 +          writer.addMutation(timeMutation);
 +      } catch (MutationsRejectedException exception) {
 +        log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception);
 +        if (log.isDebugEnabled()) {
 +          log.debug("discarded span due to rejection of mutation: " + spanMutation, exception);
 +        }
 +        /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */
 +      } catch (RuntimeException exception) {
 +        log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception);
 +        log.debug("unable to write mutation to table due to exception.", exception);
 +      }
 +    }
 +
 +  }
 +
 +  public TraceServer(ServerConfiguration serverConfiguration, String hostname) throws Exception {
 +    this.serverConfiguration = serverConfiguration;
 +    AccumuloConfiguration conf = serverConfiguration.getConfiguration();
 +    table = conf.get(Property.TRACE_TABLE);
 +    Connector connector = null;
 +    while (true) {
 +      try {
 +        String principal = conf.get(Property.TRACE_USER);
 +        AuthenticationToken at;
 +        Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
 +        if (loginMap.isEmpty()) {
 +          Property p = Property.TRACE_PASSWORD;
-           at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++          at = new PasswordToken(conf.get(p).getBytes(UTF_8));
 +        } else {
 +          Properties props = new Properties();
 +          AuthenticationToken token = AccumuloVFSClassLoader.getClassLoader().loadClass(conf.get(Property.TRACE_TOKEN_TYPE)).asSubclass(AuthenticationToken.class)
 +              .newInstance();
 +
 +          int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
 +          for (Entry<String,String> entry : loginMap.entrySet()) {
 +            props.put(entry.getKey().substring(prefixLength), entry.getValue());
 +          }
 +
 +          token.init(props);
 +
 +          at = token;
 +        }
 +
 +        connector = serverConfiguration.getInstance().getConnector(principal, at);
 +        if (!connector.tableOperations().exists(table)) {
 +          connector.tableOperations().create(table);
 +          IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName());
 +          AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
 +          connector.tableOperations().attachIterator(table, setting);
 +        }
 +        connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
 +        break;
 +      } catch (Exception ex) {
 +        log.info("Waiting to checking/create the trace table.", ex);
 +        UtilWaitThread.sleep(1000);
 +      }
 +    }
 +    this.connector = connector;
 +    // make sure we refer to the final variable from now on.
 +    connector = null;
 +
 +    int port = conf.getPort(Property.TRACE_PORT);
 +    final ServerSocket sock = ServerSocketChannel.open().socket();
 +    sock.setReuseAddress(true);
 +    sock.bind(new InetSocketAddress(hostname, port));
 +    final TServerTransport transport = new TServerSocket(sock);
 +    TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
 +    options.processor(new Processor<Iface>(new Receiver()));
 +    server = new TThreadPoolServer(options);
 +    registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort());
 +    writer = new AtomicReference<BatchWriter>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS)));
 +  }
 +
 +  public void run() throws Exception {
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        flush();
 +      }
 +    }, 1000, 1000);
 +    server.serve();
 +  }
 +
 +  private void flush() {
 +    try {
 +      final BatchWriter writer = this.writer.get();
 +      if (null != writer) {
 +        writer.flush();
 +      }
 +    } catch (MutationsRejectedException exception) {
 +      log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
 +      log.debug("flushing traces failed due to exception", exception);
 +      resetWriter();
 +      /* XXX e.g. if the writer was closed between when we grabbed it and when we called flush. */
 +    } catch (RuntimeException exception) {
 +      log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
 +      log.debug("flushing traces failed due to exception", exception);
 +      resetWriter();
 +    }
 +  }
 +
 +  private void resetWriter() {
 +    BatchWriter writer = null;
 +    try {
 +      writer = connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS));
 +    } catch (Exception ex) {
 +      log.warn("Unable to create a batch writer, will retry. Set log level to DEBUG to see stacktrace. cause: " + ex);
 +      log.debug("batch writer creation failed with exception.", ex);
 +    } finally {
 +      /* Trade in the new writer (even if null) for the one we need to close. */
 +      writer = this.writer.getAndSet(writer);
 +      try {
 +        if (null != writer) {
 +          writer.close();
 +        }
 +      } catch (Exception ex) {
 +        log.warn("Problem closing batch writer. Set log level to DEBUG to see stacktrace. cause: " + ex);
 +        log.debug("batch writer close failed with exception", ex);
 +      }
 +    }
 +  }
 +
 +  private void registerInZooKeeper(String name) throws Exception {
 +    String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-     String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(Constants.UTF8));
++    String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(UTF_8));
 +    zoo.exists(path, this);
 +  }
 +
 +  public static void main(String[] args) throws Exception {
 +    SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +    ServerOpts opts = new ServerOpts();
 +    final String app = "tracer";
 +    opts.parseArgs(app, args);
 +    Accumulo.setupLogging(app);
 +    Instance instance = HdfsZooInstance.getInstance();
 +    ServerConfiguration conf = new ServerConfiguration(instance);
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Accumulo.init(fs, conf, app);
 +    String hostname = opts.getAddress();
 +    TraceServer server = new TraceServer(conf, hostname);
 +    Accumulo.enableTracing(hostname, app);
 +    server.run();
 +    log.info("tracer stopping");
 +  }
 +
 +  @Override
 +  public void process(WatchedEvent event) {
 +    log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
 +    if (event.getState() == KeeperState.Expired) {
 +      log.warn("Trace server lost zookeeper registration at " + event.getPath());
 +      server.stop();
 +    } else if (event.getType() == EventType.NodeDeleted) {
 +      log.warn("Trace server zookeeper entry lost " + event.getPath());
 +      server.stop();
 +    }
 +    if (event.getPath() != null) {
 +      try {
 +        if (ZooReaderWriter.getInstance().exists(event.getPath(), this))
 +          return;
 +      } catch (Exception ex) {
 +        log.error(ex, ex);
 +      }
 +      log.warn("Trace server unable to reset watch on zookeeper registration");
 +      server.stop();
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index 39bf81e,0000000..4d9a6c2
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@@ -1,75 -1,0 +1,76 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.trace.TraceFileSystem;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.FileUtil;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * Copy failed bulk imports.
 + */
 +public class BulkFailedCopyProcessor implements Processor {
 +
 +  private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
 +
 +  @Override
 +  public Processor newProcessor() {
 +    return new BulkFailedCopyProcessor();
 +  }
 +
 +  @Override
 +  public void process(String workID, byte[] data) {
 +
-     String paths[] = new String(data, Constants.UTF8).split(",");
++    String paths[] = new String(data, UTF_8).split(",");
 +
 +    Path orig = new Path(paths[0]);
 +    Path dest = new Path(paths[1]);
 +    Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
 +
 +    try {
 +      VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
 +      FileSystem origFs = TraceFileSystem.wrap(vm.getVolumeByPath(orig).getFileSystem());
 +      FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
 +      
 +      FileUtil.copy(origFs, orig, destFs, tmp, false, true, CachedConfiguration.getInstance());
 +      destFs.rename(tmp, dest);
 +      log.debug("copied " + orig + " to " + dest);
 +    } catch (IOException ex) {
 +      try {
 +        VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
 +        FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
 +        destFs.create(dest).close();
 +        log.warn(" marked " + dest + " failed", ex);
 +      } catch (IOException e) {
 +        log.error("Unable to create failure flag file " + dest, e);
 +      }
 +    }
 +
 +  }
 +
 +}


[25/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 0e02f05,0000000..7158ea8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@@ -1,598 -1,0 +1,598 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.log;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_FINISH;
 +import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_START;
 +import static org.apache.accumulo.tserver.logger.LogEvents.DEFINE_TABLET;
 +import static org.apache.accumulo.tserver.logger.LogEvents.MANY_MUTATIONS;
 +import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
 +
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.EOFException;
 +import java.io.IOException;
 +import java.io.OutputStream;
 +import java.lang.reflect.InvocationTargetException;
 +import java.lang.reflect.Method;
 +import java.nio.channels.ClosedChannelException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.UUID;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.LinkedBlockingQueue;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.security.crypto.CryptoModule;
 +import org.apache.accumulo.core.security.crypto.CryptoModuleFactory;
 +import org.apache.accumulo.core.security.crypto.CryptoModuleParameters;
 +import org.apache.accumulo.core.security.crypto.DefaultCryptoModule;
 +import org.apache.accumulo.core.security.crypto.NoFlushOutputStream;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.tserver.TabletMutations;
 +import org.apache.accumulo.tserver.logger.LogFileKey;
 +import org.apache.accumulo.tserver.logger.LogFileValue;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * Wrap a connection to a logger.
 + *
 + */
 +public class DfsLogger {
 +  public static final String LOG_FILE_HEADER_V2 = "--- Log File Header (v2) ---";
 +  public static final String LOG_FILE_HEADER_V3 = "--- Log File Header (v3) ---";
 +
 +  private static Logger log = Logger.getLogger(DfsLogger.class);
 +
 +  public static class LogClosedException extends IOException {
 +    private static final long serialVersionUID = 1L;
 +
 +    public LogClosedException() {
 +      super("LogClosed");
 +    }
 +  }
 +
 +  /**
 +   * A well-timed tabletserver failure could result in an incomplete header written to a write-ahead log. This exception is thrown when the header cannot be
 +   * read from a WAL which should only happen when the tserver dies as described.
 +   */
 +  public static class LogHeaderIncompleteException extends IOException {
 +    private static final long serialVersionUID = 1l;
 +
 +    public LogHeaderIncompleteException(String msg) {
 +      super(msg);
 +    }
 +
 +    public LogHeaderIncompleteException(String msg, Throwable cause) {
 +      super(msg, cause);
 +    }
 +
 +    public LogHeaderIncompleteException(Throwable cause) {
 +      super(cause);
 +    }
 +  }
 +
 +  public static class DFSLoggerInputStreams {
 +
 +    private FSDataInputStream originalInput;
 +    private DataInputStream decryptingInputStream;
 +
 +    public DFSLoggerInputStreams(FSDataInputStream originalInput, DataInputStream decryptingInputStream) {
 +      this.originalInput = originalInput;
 +      this.decryptingInputStream = decryptingInputStream;
 +    }
 +
 +    public FSDataInputStream getOriginalInput() {
 +      return originalInput;
 +    }
 +
 +    public void setOriginalInput(FSDataInputStream originalInput) {
 +      this.originalInput = originalInput;
 +    }
 +
 +    public DataInputStream getDecryptingInputStream() {
 +      return decryptingInputStream;
 +    }
 +
 +    public void setDecryptingInputStream(DataInputStream decryptingInputStream) {
 +      this.decryptingInputStream = decryptingInputStream;
 +    }
 +  }
 +
 +  public interface ServerResources {
 +    AccumuloConfiguration getConfiguration();
 +
 +    VolumeManager getFileSystem();
 +
 +    Set<TServerInstance> getCurrentTServers();
 +  }
 +
 +  private final LinkedBlockingQueue<DfsLogger.LogWork> workQueue = new LinkedBlockingQueue<DfsLogger.LogWork>();
 +
 +  private final Object closeLock = new Object();
 +
 +  private static final DfsLogger.LogWork CLOSED_MARKER = new DfsLogger.LogWork(null);
 +
 +  private static final LogFileValue EMPTY = new LogFileValue();
 +
 +  private boolean closed = false;
 +
 +  private class LogSyncingTask implements Runnable {
 +
 +    @Override
 +    public void run() {
 +      ArrayList<DfsLogger.LogWork> work = new ArrayList<DfsLogger.LogWork>();
 +      boolean sawClosedMarker = false;
 +      while (!sawClosedMarker) {
 +        work.clear();
 +
 +        try {
 +          work.add(workQueue.take());
 +        } catch (InterruptedException ex) {
 +          continue;
 +        }
 +        workQueue.drainTo(work);
 +
 +        try {
 +          sync.invoke(logFile);
 +        } catch (Exception ex) {
 +          log.warn("Exception syncing " + ex);
 +          for (DfsLogger.LogWork logWork : work) {
 +            logWork.exception = ex;
 +          }
 +        }
 +
 +        for (DfsLogger.LogWork logWork : work)
 +          if (logWork == CLOSED_MARKER)
 +            sawClosedMarker = true;
 +          else
 +            logWork.latch.countDown();
 +      }
 +    }
 +  }
 +
 +  static class LogWork {
 +    CountDownLatch latch;
 +    volatile Exception exception;
 +
 +    public LogWork(CountDownLatch latch) {
 +      this.latch = latch;
 +    }
 +  }
 +
 +  public static class LoggerOperation {
 +    private final LogWork work;
 +
 +    public LoggerOperation(LogWork work) {
 +      this.work = work;
 +    }
 +
 +    public void await() throws IOException {
 +      try {
 +        work.latch.await();
 +      } catch (InterruptedException e) {
 +        throw new RuntimeException(e);
 +      }
 +
 +      if (work.exception != null) {
 +        if (work.exception instanceof IOException)
 +          throw (IOException) work.exception;
 +        else if (work.exception instanceof RuntimeException)
 +          throw (RuntimeException) work.exception;
 +        else
 +          throw new RuntimeException(work.exception);
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public boolean equals(Object obj) {
 +    // filename is unique
 +    if (obj == null)
 +      return false;
 +    if (obj instanceof DfsLogger)
 +      return getFileName().equals(((DfsLogger) obj).getFileName());
 +    return false;
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    // filename is unique
 +    return getFileName().hashCode();
 +  }
 +
 +  private final ServerResources conf;
 +  private FSDataOutputStream logFile;
 +  private DataOutputStream encryptingLogFile = null;
 +  private Method sync;
 +  private String logPath;
 +  private Daemon syncThread;
 +
 +  /* Track what's actually in +r/!0 for this logger ref */
 +  private String metaReference;
 +
 +  public DfsLogger(ServerResources conf) throws IOException {
 +    this.conf = conf;
 +  }
 +
 +  /**
 +   * Refernce a pre-existing log file.
 +   *
 +   * @param meta
 +   *          the cq for the "log" entry in +r/!0
 +   */
 +  public DfsLogger(ServerResources conf, String filename, String meta) throws IOException {
 +    this.conf = conf;
 +    this.logPath = filename;
 +    metaReference = meta;
 +  }
 +
 +  public static DFSLoggerInputStreams readHeaderAndReturnStream(VolumeManager fs, Path path, AccumuloConfiguration conf) throws IOException {
 +    FSDataInputStream input = fs.open(path);
 +    DataInputStream decryptingInput = null;
 +
 +    byte[] magic = DfsLogger.LOG_FILE_HEADER_V3.getBytes();
 +    byte[] magicBuffer = new byte[magic.length];
 +    try {
 +      input.readFully(magicBuffer);
 +      if (Arrays.equals(magicBuffer, magic)) {
 +        // additional parameters it needs from the underlying stream.
 +        String cryptoModuleClassname = input.readUTF();
 +        CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoModuleClassname);
 +
 +        // Create the parameters and set the input stream into those parameters
 +        CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
 +        params.setEncryptedInputStream(input);
 +
 +        // Create the plaintext input stream from the encrypted one
 +        params = cryptoModule.getDecryptingInputStream(params);
 +
 +        if (params.getPlaintextInputStream() instanceof DataInputStream) {
 +          decryptingInput = (DataInputStream) params.getPlaintextInputStream();
 +        } else {
 +          decryptingInput = new DataInputStream(params.getPlaintextInputStream());
 +        }
 +      } else {
 +        input.seek(0);
 +        byte[] magicV2 = DfsLogger.LOG_FILE_HEADER_V2.getBytes();
 +        byte[] magicBufferV2 = new byte[magicV2.length];
 +        input.readFully(magicBufferV2);
 +
 +        if (Arrays.equals(magicBufferV2, magicV2)) {
 +          // Log files from 1.5 dump their options in raw to the logger files. Since we don't know the class
 +          // that needs to read those files, we can make a couple of basic assumptions. Either it's going to be
 +          // the NullCryptoModule (no crypto) or the DefaultCryptoModule.
 +
 +          // If it's null, we won't have any parameters whatsoever. First, let's attempt to read
 +          // parameters
 +          Map<String,String> opts = new HashMap<String,String>();
 +          int count = input.readInt();
 +          for (int i = 0; i < count; i++) {
 +            String key = input.readUTF();
 +            String value = input.readUTF();
 +            opts.put(key, value);
 +          }
 +
 +          if (opts.size() == 0) {
 +            // NullCryptoModule, we're done
 +            decryptingInput = input;
 +          } else {
 +
 +            // The DefaultCryptoModule will want to read the parameters from the underlying file, so we will put the file back to that spot.
 +            org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory
 +                .getCryptoModule(DefaultCryptoModule.class.getName());
 +
 +            CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
 +
 +            input.seek(0);
 +            input.readFully(magicBufferV2);
 +            params.setEncryptedInputStream(input);
 +
 +            params = cryptoModule.getDecryptingInputStream(params);
 +            if (params.getPlaintextInputStream() instanceof DataInputStream) {
 +              decryptingInput = (DataInputStream) params.getPlaintextInputStream();
 +            } else {
 +              decryptingInput = new DataInputStream(params.getPlaintextInputStream());
 +            }
 +          }
 +
 +        } else {
 +
 +          input.seek(0);
 +          decryptingInput = input;
 +        }
 +
 +      }
 +    } catch (EOFException e) {
 +      log.warn("Got EOFException trying to read WAL header information, assuming the rest of the file (" + path + ") has no data.");
 +      // A TabletServer might have died before the (complete) header was written
 +      throw new LogHeaderIncompleteException(e);
 +    }
 +
 +    return new DFSLoggerInputStreams(input, decryptingInput);
 +  }
 +
 +  public synchronized void open(String address) throws IOException {
 +    String filename = UUID.randomUUID().toString();
 +    String logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
 +
 +    log.debug("DfsLogger.open() begin");
 +    VolumeManager fs = conf.getFileSystem();
 +
 +    logPath = fs.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
 +
 +    metaReference = toString();
 +    try {
 +      short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
 +      if (replication == 0)
 +        replication = fs.getDefaultReplication(new Path(logPath));
 +      long blockSize = conf.getConfiguration().getMemoryInBytes(Property.TSERV_WAL_BLOCKSIZE);
 +      if (blockSize == 0)
 +        blockSize = (long) (conf.getConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1);
 +      if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
 +        logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
 +      else
 +        logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
 +
 +      String syncMethod = conf.getConfiguration().get(Property.TSERV_WAL_SYNC_METHOD);
 +      try {
 +        // hsync: send data to datanodes and sync the data to disk
 +        sync = logFile.getClass().getMethod(syncMethod);
 +      } catch (Exception ex) {
 +        log.warn("Could not find configured " + syncMethod + " method, trying to fall back to old Hadoop sync method", ex);
 +
 +        try {
 +          // sync: send data to datanodes
 +          sync = logFile.getClass().getMethod("sync");
 +        } catch (Exception e) {
 +          throw new RuntimeException(e);
 +        }
 +      }
 +
 +      // Initialize the crypto operations.
 +      org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf
 +          .getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
 +
 +      // Initialize the log file with a header and the crypto params used to set up this log file.
-       logFile.write(LOG_FILE_HEADER_V3.getBytes(Constants.UTF8));
++      logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
 +
 +      CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
 +
 +      NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
 +      params.setPlaintextOutputStream(nfos);
 +
 +      // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
 +      // so that that crypto module can re-read its own parameters.
 +
 +      logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
 +
 +      params = cryptoModule.getEncryptingOutputStream(params);
 +      OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
 +
 +      // If the module just kicks back our original stream, then just use it, don't wrap it in
 +      // another data OutputStream.
 +      if (encipheringOutputStream == nfos) {
 +        log.debug("No enciphering, using raw output stream");
 +        encryptingLogFile = nfos;
 +      } else {
 +        log.debug("Enciphering found, wrapping in DataOutputStream");
 +        encryptingLogFile = new DataOutputStream(encipheringOutputStream);
 +      }
 +
 +      LogFileKey key = new LogFileKey();
 +      key.event = OPEN;
 +      key.tserverSession = filename;
 +      key.filename = filename;
 +      write(key, EMPTY);
 +      sync.invoke(logFile);
 +      log.debug("Got new write-ahead log: " + this);
 +    } catch (Exception ex) {
 +      if (logFile != null)
 +        logFile.close();
 +      logFile = null;
 +      encryptingLogFile = null;
 +      throw new IOException(ex);
 +    }
 +
 +    syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
 +    syncThread.setName("Accumulo WALog thread " + toString());
 +    syncThread.start();
 +  }
 +
 +  @Override
 +  public String toString() {
 +    String fileName = getFileName();
 +    if (fileName.contains(":"))
 +      return getLogger() + "/" + getFileName();
 +    return fileName;
 +  }
 +
 +  /**
 +   * get the cq needed to reference this logger's entry in +r/!0
 +   */
 +  public String getMeta() {
 +    if (null == metaReference) {
 +      throw new IllegalStateException("logger doesn't have meta reference. " + this);
 +    }
 +    return metaReference;
 +  }
 +
 +  public String getFileName() {
 +    return logPath.toString();
 +  }
 +
 +  public void close() throws IOException {
 +
 +    synchronized (closeLock) {
 +      if (closed)
 +        return;
 +      // after closed is set to true, nothing else should be added to the queue
 +      // CLOSED_MARKER should be the last thing on the queue, therefore when the
 +      // background thread sees the marker and exits there should be nothing else
 +      // to process... so nothing should be left waiting for the background
 +      // thread to do work
 +      closed = true;
 +      workQueue.add(CLOSED_MARKER);
 +    }
 +
 +    // wait for background thread to finish before closing log file
 +    if (syncThread != null) {
 +      try {
 +        syncThread.join();
 +      } catch (InterruptedException e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +
 +    // expect workq should be empty at this point
 +    if (workQueue.size() != 0) {
 +      log.error("WAL work queue not empty after sync thread exited");
 +      throw new IllegalStateException("WAL work queue not empty after sync thread exited");
 +    }
 +
 +    if (encryptingLogFile != null)
 +      try {
 +        logFile.close();
 +      } catch (IOException ex) {
 +        log.error(ex);
 +        throw new LogClosedException();
 +      }
 +  }
 +
 +  public synchronized void defineTablet(int seq, int tid, KeyExtent tablet) throws IOException {
 +    // write this log to the METADATA table
 +    final LogFileKey key = new LogFileKey();
 +    key.event = DEFINE_TABLET;
 +    key.seq = seq;
 +    key.tid = tid;
 +    key.tablet = tablet;
 +    try {
 +      write(key, EMPTY);
 +      sync.invoke(logFile);
 +    } catch (IllegalArgumentException e) {
 +      log.error("Signature of sync method changed. Accumulo is likely incompatible with this version of Hadoop.");
 +      throw new RuntimeException(e);
 +    } catch (IllegalAccessException e) {
 +      log.error("Could not invoke sync method due to permission error.");
 +      throw new RuntimeException(e);
 +    } catch (InvocationTargetException e) {
 +      Throwable cause = e.getCause();
 +      if (cause instanceof IOException) {
 +        throw (IOException) cause;
 +      } else if (cause instanceof RuntimeException) {
 +        throw (RuntimeException) cause;
 +      } else if (cause instanceof Error) {
 +        throw (Error) cause;
 +      } else {
 +        // Cause is null, or some other checked exception that was added later.
 +        throw new RuntimeException(e);
 +      }
 +    }
 +  }
 +
 +  private synchronized void write(LogFileKey key, LogFileValue value) throws IOException {
 +    key.write(encryptingLogFile);
 +    value.write(encryptingLogFile);
 +    encryptingLogFile.flush();
 +  }
 +
 +  public LoggerOperation log(int seq, int tid, Mutation mutation) throws IOException {
 +    return logManyTablets(Collections.singletonList(new TabletMutations(tid, seq, Collections.singletonList(mutation))));
 +  }
 +
 +  private LoggerOperation logFileData(List<Pair<LogFileKey,LogFileValue>> keys) throws IOException {
 +    DfsLogger.LogWork work = new DfsLogger.LogWork(new CountDownLatch(1));
 +    synchronized (DfsLogger.this) {
 +      try {
 +        for (Pair<LogFileKey,LogFileValue> pair : keys) {
 +          write(pair.getFirst(), pair.getSecond());
 +        }
 +      } catch (ClosedChannelException ex) {
 +        throw new LogClosedException();
 +      } catch (Exception e) {
 +        log.error(e, e);
 +        work.exception = e;
 +      }
 +    }
 +
 +    synchronized (closeLock) {
 +      // use a different lock for close check so that adding to work queue does not need
 +      // to wait on walog I/O operations
 +
 +      if (closed)
 +        throw new LogClosedException();
 +      workQueue.add(work);
 +    }
 +
 +    return new LoggerOperation(work);
 +  }
 +
 +  public LoggerOperation logManyTablets(List<TabletMutations> mutations) throws IOException {
 +    List<Pair<LogFileKey,LogFileValue>> data = new ArrayList<Pair<LogFileKey,LogFileValue>>();
 +    for (TabletMutations tabletMutations : mutations) {
 +      LogFileKey key = new LogFileKey();
 +      key.event = MANY_MUTATIONS;
 +      key.seq = tabletMutations.getSeq();
 +      key.tid = tabletMutations.getTid();
 +      LogFileValue value = new LogFileValue();
 +      value.mutations = tabletMutations.getMutations();
 +      data.add(new Pair<LogFileKey,LogFileValue>(key, value));
 +    }
 +    return logFileData(data);
 +  }
 +
 +  public LoggerOperation minorCompactionFinished(int seq, int tid, String fqfn) throws IOException {
 +    LogFileKey key = new LogFileKey();
 +    key.event = COMPACTION_FINISH;
 +    key.seq = seq;
 +    key.tid = tid;
 +    return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
 +  }
 +
 +  public LoggerOperation minorCompactionStarted(int seq, int tid, String fqfn) throws IOException {
 +    LogFileKey key = new LogFileKey();
 +    key.event = COMPACTION_START;
 +    key.seq = seq;
 +    key.tid = tid;
 +    key.filename = fqfn;
 +    return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
 +  }
 +
 +  public String getLogger() {
 +    String parts[] = logPath.split("/");
 +    return StringUtil.join(Arrays.asList(parts[parts.length - 2].split("[+]")), ":");
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
index 997f71b,0000000..2c807fd
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
@@@ -1,96 -1,0 +1,97 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.logger;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.DataInput;
 +import java.io.DataOutput;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.server.data.ServerMutation;
 +import org.apache.hadoop.io.Writable;
 +
 +public class LogFileValue implements Writable {
 +  
 +  private static final List<Mutation> empty = Collections.emptyList();
 +  
 +  public List<Mutation> mutations = empty;
 +  
 +  @Override
 +  public void readFields(DataInput in) throws IOException {
 +    int count = in.readInt();
 +    mutations = new ArrayList<Mutation>(count);
 +    for (int i = 0; i < count; i++) {
 +      ServerMutation mutation = new ServerMutation();
 +      mutation.readFields(in);
 +      mutations.add(mutation);
 +    }
 +  }
 +  
 +  @Override
 +  public void write(DataOutput out) throws IOException {
 +    out.writeInt(mutations.size());
 +    for (Mutation m : mutations) {
 +      m.write(out);
 +    }
 +  }
 +  
 +  public static void print(LogFileValue value) {
 +    System.out.println(value.toString());
 +  }
 +  
 +  private static String displayLabels(byte[] labels) {
-     String s = new String(labels, Constants.UTF8);
++    String s = new String(labels, UTF_8);
 +    s = s.replace("&", " & ");
 +    s = s.replace("|", " | ");
 +    return s;
 +  }
 +  
 +  public static String format(LogFileValue lfv, int maxMutations) {
 +    if (lfv.mutations.size() == 0)
 +      return "";
 +    StringBuilder builder = new StringBuilder();
 +    builder.append(lfv.mutations.size() + " mutations:\n");
 +    int i = 0;
 +    for (Mutation m : lfv.mutations) {
 +      if (i++ >= maxMutations) {
 +        builder.append("...");
 +        break;
 +      }
-       builder.append("  ").append(new String(m.getRow(), Constants.UTF8)).append("\n");
++      builder.append("  ").append(new String(m.getRow(), UTF_8)).append("\n");
 +      for (ColumnUpdate update : m.getUpdates()) {
 +        String value = new String(update.getValue());
-         builder.append("      ").append(new String(update.getColumnFamily(), Constants.UTF8)).append(":")
-                 .append(new String(update.getColumnQualifier(), Constants.UTF8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
++        builder.append("      ").append(new String(update.getColumnFamily(), UTF_8)).append(":")
++                .append(new String(update.getColumnQualifier(), UTF_8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
 +                .append(update.getTimestamp()).append(" [").append(displayLabels(update.getColumnVisibility())).append("] ")
 +                .append(update.isDeleted() ? "<deleted>" : value).append("\n");
 +      }
 +    }
 +    return builder.toString();
 +  }
 +  
 +  @Override
 +  public String toString() {
 +    return format(this, 5);
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index c3f4fd0,0000000..29eefc8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -1,178 -1,0 +1,179 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.logger;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.DataInputStream;
 +import java.io.EOFException;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Set;
 +import java.util.regex.Matcher;
 +import java.util.regex.Pattern;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
 +import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException;
 +import org.apache.accumulo.tserver.log.MultiReader;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.JCommander;
 +import com.beust.jcommander.Parameter;
 +
 +public class LogReader {
 +  private static final Logger log = Logger.getLogger(LogReader.class);
 +
 +  static class Opts extends Help {
 +    @Parameter(names = "-r", description = "print only mutations associated with the given row")
 +    String row;
 +    @Parameter(names = "-m", description = "limit the number of mutations printed per row")
 +    int maxMutations = 5;
 +    @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
 +    String extent;
 +    @Parameter(names = "-p", description = "search for a row that matches the given regex")
 +    String regexp;
 +    @Parameter(description = "<logfile> { <logfile> ... }")
 +    List<String> files = new ArrayList<String>();
 +  }
 +
 +  /**
 +   * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
 +   *
 +   * @param args
 +   *          - first argument is the file to print
 +   */
 +  public static void main(String[] args) throws IOException {
 +    Opts opts = new Opts();
 +    opts.parseArgs(LogReader.class.getName(), args);
 +    VolumeManager fs = VolumeManagerImpl.get();
 +
 +    Matcher rowMatcher = null;
 +    KeyExtent ke = null;
 +    Text row = null;
 +    if (opts.files.isEmpty()) {
 +      new JCommander(opts).usage();
 +      return;
 +    }
 +    if (opts.row != null)
 +      row = new Text(opts.row);
 +    if (opts.extent != null) {
 +      String sa[] = opts.extent.split(";");
 +      ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
 +    }
 +    if (opts.regexp != null) {
 +      Pattern pattern = Pattern.compile(opts.regexp);
 +      rowMatcher = pattern.matcher("");
 +    }
 +
 +    Set<Integer> tabletIds = new HashSet<Integer>();
 +
 +    for (String file : opts.files) {
 +
 +      Path path = new Path(file);
 +      LogFileKey key = new LogFileKey();
 +      LogFileValue value = new LogFileValue();
 +
 +      if (fs.isFile(path)) {
 +        // read log entries from a simple hdfs file
 +        DFSLoggerInputStreams streams;
 +        try {
 +          streams = DfsLogger.readHeaderAndReturnStream(fs, path, ServerConfiguration.getSiteConfiguration());
 +        } catch (LogHeaderIncompleteException e) {
 +          log.warn("Could not read header for " + path + ". Ignoring...");
 +          continue;
 +        }
 +        DataInputStream input = streams.getDecryptingInputStream();
 +
 +        try {
 +          while (true) {
 +            try {
 +              key.readFields(input);
 +              value.readFields(input);
 +            } catch (EOFException ex) {
 +              break;
 +            }
 +            printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
 +          }
 +        } finally {
 +          input.close();
 +        }
 +      } else {
 +        // read the log entries sorted in a map file
 +        MultiReader input = new MultiReader(fs, path);
 +        while (input.next(key, value)) {
 +          printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
 +        }
 +      }
 +    }
 +  }
 +
 +  public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
 +
 +    if (ke != null) {
 +      if (key.event == LogEvents.DEFINE_TABLET) {
 +        if (key.tablet.equals(ke)) {
 +          tabletIds.add(key.tid);
 +        } else {
 +          return;
 +        }
 +      } else if (!tabletIds.contains(key.tid)) {
 +        return;
 +      }
 +    }
 +
 +    if (row != null || rowMatcher != null) {
 +      if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
 +        boolean found = false;
 +        for (Mutation m : value.mutations) {
 +          if (row != null && new Text(m.getRow()).equals(row)) {
 +            found = true;
 +            break;
 +          }
 +
 +          if (rowMatcher != null) {
-             rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
++            rowMatcher.reset(new String(m.getRow(), UTF_8));
 +            if (rowMatcher.matches()) {
 +              found = true;
 +              break;
 +            }
 +          }
 +        }
 +
 +        if (!found)
 +          return;
 +      } else {
 +        return;
 +      }
 +
 +    }
 +
 +    System.out.println(key);
 +    System.out.println(LogFileValue.format(value, maxMutations));
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
index a679c48,ef6311b..3c20858
--- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
@@@ -71,98 -72,71 +72,98 @@@ public class TestBinaryRows 
      return l;
    }
    
 -  static class Opts extends ClientOnRequiredTable {
 +  public static class Opts extends ClientOnRequiredTable {
      @Parameter(names="--mode", description="either 'ingest', 'delete', 'randomLookups', 'split', 'verify', 'verifyDeleted'", required=true)
 -    String mode;
 +    public String mode;
      @Parameter(names="--start", description="the lowest numbered row")
 -    long start = 0;
 +    public long start = 0;
      @Parameter(names="--count", description="number of rows to ingest", required=true)
 -    long num = 0;
 +    public long num = 0;
    }
    
 -  public static void main(String[] args) {
 -    Opts opts = new Opts();
 -    BatchWriterOpts bwOpts = new BatchWriterOpts();
 -    ScannerOpts scanOpts = new ScannerOpts();
 -    opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
 +  public static void runTest(Connector connector, Opts opts, BatchWriterOpts bwOpts, ScannerOpts scanOpts) throws Exception {
      
 -    try {
 -      Connector connector = opts.getConnector();
 -      
 -      final Text CF = new Text("cf"), CQ = new Text("cq");
 -      final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
 +    final Text CF = new Text("cf"), CQ = new Text("cq");
-     final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
++    final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
 +    if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
 +      BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
 +      boolean delete = opts.mode.equals("delete");
        
 -      if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
 -        BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
 -        boolean delete = opts.mode.equals("delete");
 -        
 -        for (long i = 0; i < opts.num; i++) {
 -          byte[] row = encodeLong(i + opts.start);
 -          String value = "" + (i + opts.start);
 -          
 -          Mutation m = new Mutation(new Text(row));
 -          if (delete) {
 -            m.putDelete(CF, CQ);
 -          } else {
 -            m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
 -          }
 -          bw.addMutation(m);
 +      for (long i = 0; i < opts.num; i++) {
 +        byte[] row = encodeLong(i + opts.start);
 +        String value = "" + (i + opts.start);
 +        
 +        Mutation m = new Mutation(new Text(row));
 +        if (delete) {
 +          m.putDelete(CF, CQ);
 +        } else {
-           m.put(CF, CQ, new Value(value.getBytes(Constants.UTF8)));
++          m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
          }
 -        
 -        bw.close();
 -      } else if (opts.mode.equals("verifyDeleted")) {
 -        Scanner s = connector.createScanner(opts.tableName, opts.auths);
 -        s.setBatchSize(scanOpts.scanBatchSize);
 -        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 -        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
 -        s.setBatchSize(50000);
 -        s.setRange(new Range(startKey, stopKey));
 -        
 -        for (Entry<Key,Value> entry : s) {
 -          System.err.println("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
 -          System.err.println("exiting...");
 -          System.exit(1);
 -        }
 -        
 -      } else if (opts.mode.equals("verify")) {
 -        long t1 = System.currentTimeMillis();
 +        bw.addMutation(m);
 +      }
 +      
 +      bw.close();
 +    } else if (opts.mode.equals("verifyDeleted")) {
 +      Scanner s = connector.createScanner(opts.tableName, opts.auths);
 +      s.setBatchSize(scanOpts.scanBatchSize);
 +      Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 +      Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
 +      s.setBatchSize(50000);
 +      s.setRange(new Range(startKey, stopKey));
 +      
 +      for (Entry<Key,Value> entry : s) {
 +        throw new Exception("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
 +      }
 +      
 +    } else if (opts.mode.equals("verify")) {
 +      long t1 = System.currentTimeMillis();
 +      
 +      Scanner s = connector.createScanner(opts.tableName, opts.auths);
 +      Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 +      Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
 +      s.setBatchSize(scanOpts.scanBatchSize);
 +      s.setRange(new Range(startKey, stopKey));
 +      
 +      long i = opts.start;
 +      
 +      for (Entry<Key,Value> e : s) {
 +        Key k = e.getKey();
 +        Value v = e.getValue();
 +
 +        checkKeyValue(i, k, v);
 +
 +        i++;
 +      }
 +      
 +      if (i != opts.start + opts.num) {
 +        throw new Exception("ERROR : did not see expected number of rows, saw " + (i - opts.start) + " expected " + opts.num);
 +      }
 +      
 +      long t2 = System.currentTimeMillis();
 +      
 +      System.out.printf("time : %9.2f secs%n", ((t2 - t1) / 1000.0));
 +      System.out.printf("rate : %9.2f entries/sec%n", opts.num / ((t2 - t1) / 1000.0));
 +      
 +    } else if (opts.mode.equals("randomLookups")) {
 +      int numLookups = 1000;
 +      
 +      Random r = new Random();
 +      
 +      long t1 = System.currentTimeMillis();
 +      
 +      for (int i = 0; i < numLookups; i++) {
 +        long row = ((r.nextLong() & 0x7fffffffffffffffl) % opts.num) + opts.start;
          
          Scanner s = connector.createScanner(opts.tableName, opts.auths);
 -        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 -        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
          s.setBatchSize(scanOpts.scanBatchSize);
 +        Key startKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 +        Key stopKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], 0);
          s.setRange(new Range(startKey, stopKey));
          
 -        long i = opts.start;
 +        Iterator<Entry<Key,Value>> si = s.iterator();
          
 -        for (Entry<Key,Value> e : s) {
 +        if (si.hasNext()) {
 +          Entry<Key,Value> e = si.next();
            Key k = e.getKey();
            Value v = e.getValue();
            

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestIngest.java
index bc3ff05,5d8e268..c3e77af
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@@ -16,16 -16,14 +16,17 @@@
   */
  package org.apache.accumulo.test;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 +import java.io.IOException;
  import java.util.Map.Entry;
  import java.util.Random;
  import java.util.Set;
  import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.BatchWriterOpts;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.BatchWriter;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index 7a848a3,125ef5f..16f0b3f
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@@ -16,11 -16,11 +16,12 @@@
   */
  package org.apache.accumulo.test;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.util.ArrayList;
 +import java.util.List;
  import java.util.Map.Entry;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.BatchWriterOpts;
  import org.apache.accumulo.core.cli.ScannerOpts;
  import org.apache.accumulo.core.client.AccumuloException;
@@@ -101,8 -97,8 +102,8 @@@ public class TestMultiTableIngest 
        
        // populate
        for (int i = 0; i < opts.count; i++) {
 -        Mutation m = new Mutation(new Text(String.format("%05d", i)));
 +        Mutation m = new Mutation(new Text(String.format("%06d", i)));
-         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
+         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
          b.getBatchWriter(tableNames.get(i % tableNames.size())).addMutation(m);
        }
        try {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
index 6082ebe,5962220..4f88c1b
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
@@@ -26,7 -28,7 +28,6 @@@ import java.util.Map.Entry
  import java.util.Random;
  import java.util.zip.CRC32;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Scanner;
  import org.apache.accumulo.core.data.Key;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
index 4bd07ca,6124a3d..11d19c5
--- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
@@@ -82,9 -82,9 +83,9 @@@ public class UndefinedAnalyzer 
          parseLog(log);
        }
      }
 -    
 +
      private void parseLog(File log) throws Exception {
-       BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), Constants.UTF8));
+       BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), UTF_8));
        String line;
        TreeMap<Long,Long> tm = null;
        try {
@@@ -173,10 -172,10 +174,10 @@@
        SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM");
        String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + "";
        String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + "";
 -      
 +
        for (File masterLog : masterLogs) {
 -        
 +
-         BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), Constants.UTF8));
+         BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), UTF_8));
          String line;
          try {
            while ((line = reader.readLine()) != null) {
@@@ -256,11 -252,11 +257,11 @@@
    public static void main(String[] args) throws Exception {
      Opts opts = new Opts();
      BatchScannerOpts bsOpts = new BatchScannerOpts();
 -    opts.parseArgs(UndefinedAnalyzer.class.getName(), args, opts);
 -    
 +    opts.parseArgs(UndefinedAnalyzer.class.getName(), args, bsOpts);
 +
      List<UndefinedNode> undefs = new ArrayList<UndefinedNode>();
 -    
 +
-     BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, Constants.UTF8));
+     BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, UTF_8));
      String line;
      while ((line = reader.readLine()) != null) {
        String[] tokens = line.split("\\s");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index f26c8d7,166e2ad..949fc52
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -16,6 -16,10 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import java.net.InetAddress;
 -import java.net.InetSocketAddress;
  import java.util.HashMap;
  import java.util.Random;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index cb17340,33ef0b5..72efee3
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -24,7 -26,7 +26,6 @@@ import java.util.Random
  import java.util.TreeSet;
  import java.util.UUID;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.BatchScanner;
  import org.apache.accumulo.core.client.BatchWriter;
  import org.apache.accumulo.core.client.BatchWriterConfig;
@@@ -104,10 -100,10 +105,10 @@@ public class MetadataBatchScanTest 
          
          String dir = "/t-" + UUID.randomUUID();
          
-         TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
 -        Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
++        TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
          
          for (int i = 0; i < 5; i++) {
-           mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
 -          mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
++          mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
          }
          
          bw.addMutation(mut);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
index 5756934,0191b50..e4ce3f3
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
@@@ -41,9 -37,7 +43,8 @@@ import javax.xml.parsers.DocumentBuilde
  import javax.xml.validation.Schema;
  import javax.xml.validation.SchemaFactory;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.util.SimpleThreadPool;
  import org.apache.log4j.Level;
  import org.w3c.dom.Document;
  import org.w3c.dom.Element;
@@@ -201,163 -193,109 +202,163 @@@ public class Module extends Node 
      if (fixture != null) {
        fixture.setUp(state);
      }
 -    
 -    Node initNode = getNode(initNodeId);
 -    
 -    boolean test = false;
 -    if (initNode instanceof Test) {
 -      startTimer(initNode);
 -      test = true;
 -    }
 -    initNode.visit(state, getProps(initNodeId));
 -    if (test)
 -      stopTimer(initNode);
 -    
 -    state.visitedNode();
 -    // update aliases
 -    Set<String> aliases;
 -    if ((aliases = aliasMap.get(initNodeId)) != null)
 -      for (String alias : aliases) {
 -        ((Alias) nodes.get(alias)).update(initNodeId);
 -      }
 -    
 -    String curNodeId = initNodeId;
 -    int numHops = 0;
 -    long startTime = System.currentTimeMillis() / 1000;
 -    while (true) {
 -      // check if END state was reached
 -      if (curNodeId.equalsIgnoreCase("END")) {
 -        log.debug("reached END state");
 -        break;
 -      }
 -      // check if maxSec was reached
 -      long curTime = System.currentTimeMillis() / 1000;
 -      if ((curTime - startTime) > maxSec) {
 -        log.debug("reached maxSec(" + maxSec + ")");
 -        break;
 -      }
 -      // check if maxHops was reached
 -      if (numHops > maxHops) {
 -        log.debug("reached maxHops(" + maxHops + ")");
 -        break;
 -      }
 -      numHops++;
 -      
 -      if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
 -        throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
 -      }
 -      AdjList adj = adjMap.get(curNodeId);
 -      String nextNodeId = adj.randomNeighbor();
 -      Node nextNode = getNode(nextNodeId);
 -      if (nextNode instanceof Alias) {
 -        nextNodeId = ((Alias) nextNode).getTargetId();
 -        nextNode = ((Alias) nextNode).get();
 +
 +    ExecutorService service = new SimpleThreadPool(1, "RandomWalk Runner");
 +
 +    try {
 +      Node initNode = getNode(initNodeId);
 +
 +      boolean test = false;
 +      if (initNode instanceof Test) {
 +        startTimer(initNode);
 +        test = true;
        }
 -      Properties nodeProps = getProps(nextNodeId);
 -      try {
 -        test = false;
 -        if (nextNode instanceof Test) {
 -          startTimer(nextNode);
 -          test = true;
 +      initNode.visit(state, getProps(initNodeId));
 +      if (test)
 +        stopTimer(initNode);
 +
 +      state.visitedNode();
 +      // update aliases
 +      Set<String> aliases;
 +      if ((aliases = aliasMap.get(initNodeId)) != null)
 +        for (String alias : aliases) {
 +          ((Alias) nodes.get(alias)).update(initNodeId);
          }
 -        nextNode.visit(state, nodeProps);
 -        if (test)
 -          stopTimer(nextNode);
 -      } catch (Exception e) {
 -        log.debug("Connector belongs to user: " + state.getConnector().whoami());
 -        log.debug("Exception occured at: " + System.currentTimeMillis());
 -        log.debug("Properties for node: " + nextNodeId);
 -        for (Entry<Object,Object> entry : nodeProps.entrySet()) {
 -          log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +
 +      String curNodeId = initNodeId;
 +      int numHops = 0;
 +      long startTime = System.currentTimeMillis() / 1000;
 +      while (true) {
 +        // check if END state was reached
 +        if (curNodeId.equalsIgnoreCase("END")) {
 +          log.debug("reached END state");
 +          break;
          }
 -        log.debug("Overall Properties");
 -        for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
 -          log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +        // check if maxSec was reached
 +        long curTime = System.currentTimeMillis() / 1000;
 +        if ((curTime - startTime) > maxSec) {
 +          log.debug("reached maxSec(" + maxSec + ")");
 +          break;
          }
 -        log.debug("State information");
 -        for (String key : new TreeSet<String>(state.getMap().keySet()))  {
 -          Object value = state.getMap().get(key);
 -          String logMsg = "  " + key + ": ";
 -          if (value == null)
 -            logMsg += "null";
 -          else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
 -            logMsg += value;
 -          else if (value instanceof byte[])
 -            logMsg += new String((byte[])value, UTF_8);
 -          else if (value instanceof PasswordToken)
 -            logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
 -          else
 -            logMsg += value.getClass()+ " - " + value;
 -          
 -          log.debug(logMsg);
 +
 +        // The number of seconds before the test should exit
 +        long secondsRemaining = maxSec - (curTime - startTime);
 +
 +        // check if maxHops was reached
 +        if (numHops > maxHops) {
 +          log.debug("reached maxHops(" + maxHops + ")");
 +          break;
          }
 -        throw new Exception("Error running node " + nextNodeId, e);
 -      }
 -      state.visitedNode();
 -      
 -      // update aliases
 -      if ((aliases = aliasMap.get(curNodeId)) != null)
 -        for (String alias : aliases) {
 -          ((Alias) nodes.get(alias)).update(curNodeId);
 +        numHops++;
 +
 +        if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
 +          throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
 +        }
 +        AdjList adj = adjMap.get(curNodeId);
 +        String nextNodeId = adj.randomNeighbor();
 +        final Node nextNode;
 +        Node nextNodeOrAlias = getNode(nextNodeId);
 +        if (nextNodeOrAlias instanceof Alias) {
 +          nextNodeId = ((Alias) nextNodeOrAlias).getTargetId();
 +          nextNode = ((Alias) nextNodeOrAlias).get();
 +        } else {
 +          nextNode = nextNodeOrAlias;
 +        }
 +        final Properties nodeProps = getProps(nextNodeId);
 +        try {
 +          test = false;
 +          if (nextNode instanceof Test) {
 +            startTimer(nextNode);
 +            test = true;
 +          }
 +
 +          // Wrap the visit of the next node in the module in a callable that returns a thrown exception
 +          FutureTask<Exception> task = new FutureTask<Exception>(new Callable<Exception>() {
 +
 +            @Override
 +            public Exception call() throws Exception {
 +              try {
 +                nextNode.visit(state, nodeProps);
 +                return null;
 +              } catch (Exception e) {
 +                return e;
 +              }
 +            }
 +
 +          });
 +
 +          // Run the task (should execute immediately)
 +          service.submit(task);
 +
 +          Exception nodeException;
 +          try {
 +            // Bound the time we'll wait for the node to complete
 +            nodeException = task.get(secondsRemaining, TimeUnit.SECONDS);
 +          } catch (InterruptedException e) {
 +            log.warn("Interrupted waiting for " + nextNode.getClass().getSimpleName() + " to complete. Exiting.", e);
 +            break;
 +          } catch (ExecutionException e) {
 +            log.error("Caught error executing " + nextNode.getClass().getSimpleName(), e);
 +            throw e;
 +          } catch (TimeoutException e) {
 +            log.info("Timed out waiting for " + nextNode.getClass().getSimpleName() + " to complete (waited " + secondsRemaining + " seconds). Exiting.", e);
 +            break;
 +          }
 +
 +          // The RandomWalk node throw an Exception that that Callable handed back
 +          // Throw it and let the Module perform cleanup
 +          if (null != nodeException) {
 +            throw nodeException;
 +          }
 +
 +          if (test)
 +            stopTimer(nextNode);
 +        } catch (Exception e) {
 +          log.debug("Connector belongs to user: " + state.getConnector().whoami());
 +          log.debug("Exception occured at: " + System.currentTimeMillis());
 +          log.debug("Properties for node: " + nextNodeId);
 +          for (Entry<Object,Object> entry : nodeProps.entrySet()) {
 +            log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +          }
 +          log.debug("Overall Properties");
 +          for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
 +            log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +          }
 +          log.debug("State information");
 +          for (String key : new TreeSet<String>(state.getMap().keySet())) {
 +            Object value = state.getMap().get(key);
 +            String logMsg = "  " + key + ": ";
 +            if (value == null)
 +              logMsg += "null";
 +            else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
 +              logMsg += value;
 +            else if (value instanceof byte[])
-               logMsg += new String((byte[]) value, Constants.UTF8);
++              logMsg += new String((byte[]) value, UTF_8);
 +            else if (value instanceof PasswordToken)
-               logMsg += new String(((PasswordToken) value).getPassword(), Constants.UTF8);
++              logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
 +            else
 +              logMsg += value.getClass() + " - " + value;
 +
 +            log.debug(logMsg);
 +          }
 +          throw new Exception("Error running node " + nextNodeId, e);
          }
 -      
 -      curNodeId = nextNodeId;
 +        state.visitedNode();
 +
 +        // update aliases
 +        if ((aliases = aliasMap.get(curNodeId)) != null)
 +          for (String alias : aliases) {
 +            ((Alias) nodes.get(alias)).update(curNodeId);
 +          }
 +
 +        curNodeId = nextNodeId;
 +      }
 +    } finally {
 +      if (null != service) {
 +        service.shutdownNow();
 +      }
      }
 -    
 +
      if (teardown && (fixture != null)) {
        log.debug("tearing down module");
        fixture.tearDown(state);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
index 1704e49,05cc4f0..67438d9
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
@@@ -20,10 -21,10 +21,10 @@@ import static com.google.common.base.Ch
  import org.apache.accumulo.core.data.Value;
  import org.apache.accumulo.test.randomwalk.State;
  
 -public class BulkMinusOne extends BulkTest {
 -  
 +public class BulkMinusOne extends BulkImportTest {
 +
-   private static final Value negOne = new Value("-1".getBytes(Constants.UTF8));
+   private static final Value negOne = new Value("-1".getBytes(UTF_8));
 -  
 +
    @Override
    protected void runLater(State state) throws Exception {
      log.info("Decrementing");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
index f002274,7f9a218..3dcff6e
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
@@@ -51,10 -52,10 +52,10 @@@ public class BatchWrite extends Test 
        try {
          int numRows = rand.nextInt(100000);
          for (int i = 0; i < numRows; i++) {
 -          Mutation m = new Mutation(String.format("%016x", (rand.nextLong() & 0x7fffffffffffffffl)));
 -          long val = (rand.nextLong() & 0x7fffffffffffffffl);
 +          Mutation m = new Mutation(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl));
 +          long val = rand.nextLong() & 0x7fffffffffffffffl;
            for (int j = 0; j < 10; j++) {
-             m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(Constants.UTF8)));
+             m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(UTF_8)));
            }
            
            bw.addMutation(m);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
index c8da3d8,c227a11..b3519b6
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
@@@ -47,8 -50,8 +49,8 @@@ public class StopTabletServer extends T
            Collections.sort(children);
            Stat stat = new Stat();
            byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
-           if (!"master".equals(new String(data, Constants.UTF8))) {
+           if (!"master".equals(new String(data, UTF_8))) {
 -            result.add(new TServerInstance(AddressUtil.parseAddress(child, Property.TSERV_CLIENTPORT), stat.getEphemeralOwner()));
 +            result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
            }
          }
        } catch (KeeperException.NoNodeException ex) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------


[31/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index 9537f4e,0000000..83e83cf
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -1,1315 -1,0 +1,1317 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicInteger;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.ThriftTransportPool;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.master.thrift.MasterClientService.Iface;
 +import org.apache.accumulo.core.master.thrift.MasterClientService.Processor;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.master.thrift.MasterState;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.AgeOffStore;
 +import org.apache.accumulo.fate.Fate;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.master.recovery.RecoveryManager;
 +import org.apache.accumulo.master.state.TableCounts;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.init.Initialize;
 +import org.apache.accumulo.server.master.LiveTServerSet;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
 +import org.apache.accumulo.server.master.balancer.TabletBalancer;
 +import org.apache.accumulo.server.master.state.CurrentState;
 +import org.apache.accumulo.server.master.state.DeadServerList;
 +import org.apache.accumulo.server.master.state.MergeInfo;
 +import org.apache.accumulo.server.master.state.MergeState;
 +import org.apache.accumulo.server.master.state.MetaDataStateStore;
 +import org.apache.accumulo.server.master.state.RootTabletStateStore;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.state.TabletLocationState;
 +import org.apache.accumulo.server.master.state.TabletMigration;
 +import org.apache.accumulo.server.master.state.TabletState;
 +import org.apache.accumulo.server.master.state.ZooStore;
 +import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.security.handler.ZKPermHandler;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.accumulo.server.tables.TableObserver;
 +import org.apache.accumulo.server.util.DefaultMap;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.RpcWrapper;
 +import org.apache.accumulo.server.util.TServerUtils;
 +import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.start.classloader.vfs.ContextManager;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.DataInputBuffer;
 +import org.apache.hadoop.io.DataOutputBuffer;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.server.TServer;
 +import org.apache.thrift.transport.TTransportException;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.google.common.collect.Iterables;
 +
 +/**
 + * The Master is responsible for assigning and balancing tablets to tablet servers.
 + *
 + * The master will also coordinate log recoveries and reports general status.
 + */
 +public class Master implements LiveTServerSet.Listener, TableObserver, CurrentState {
 +
 +  final static Logger log = Logger.getLogger(Master.class);
 +
 +  final static int ONE_SECOND = 1000;
 +  final private static Text METADATA_TABLE_ID = new Text(MetadataTable.ID);
 +  final private static Text ROOT_TABLE_ID = new Text(RootTable.ID);
 +  final static long TIME_TO_WAIT_BETWEEN_SCANS = 60 * ONE_SECOND;
 +  final private static long TIME_BETWEEN_MIGRATION_CLEANUPS = 5 * 60 * ONE_SECOND;
 +  final static long WAIT_BETWEEN_ERRORS = ONE_SECOND;
 +  final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;
 +  final private static int MAX_CLEANUP_WAIT_TIME = ONE_SECOND;
 +  final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = ONE_SECOND;
 +  final static int MAX_TSERVER_WORK_CHUNK = 5000;
 +  final private static int MAX_BAD_STATUS_COUNT = 3;
 +
 +  final VolumeManager fs;
 +  final private Instance instance;
 +  final private String hostname;
 +  final LiveTServerSet tserverSet;
 +  final private List<TabletGroupWatcher> watchers = new ArrayList<TabletGroupWatcher>();
 +  final SecurityOperation security;
 +  final Map<TServerInstance,AtomicInteger> badServers = Collections.synchronizedMap(new DefaultMap<TServerInstance,AtomicInteger>(new AtomicInteger()));
 +  final Set<TServerInstance> serversToShutdown = Collections.synchronizedSet(new HashSet<TServerInstance>());
 +  final SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,TServerInstance>());
 +  final EventCoordinator nextEvent = new EventCoordinator();
 +  final private Object mergeLock = new Object();
 +  RecoveryManager recoveryManager = null;
 +
 +  ZooLock masterLock = null;
 +  private TServer clientService = null;
 +  TabletBalancer tabletBalancer;
 +
 +  private MasterState state = MasterState.INITIAL;
 +
 +  Fate<Master> fate;
 +
 +  volatile SortedMap<TServerInstance,TabletServerStatus> tserverStatus = Collections.unmodifiableSortedMap(new TreeMap<TServerInstance,TabletServerStatus>());
 +
 +  synchronized MasterState getMasterState() {
 +    return state;
 +  }
 +
 +  public boolean stillMaster() {
 +    return getMasterState() != MasterState.STOP;
 +  }
 +
 +  static final boolean X = true;
 +  static final boolean _ = false;
 +  // @formatter:off
 +  static final boolean transitionOK[][] = {
 +      //                              INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
 +      /* INITIAL */                   {X,     X,        _,        _,      _,         _,          X},
 +      /* HAVE_LOCK */                 {_,     X,        X,        X,      _,         _,          X},
 +      /* SAFE_MODE */                 {_,     _,        X,        X,      X,         _,          X},
 +      /* NORMAL */                    {_,     _,        X,        X,      X,         _,          X},
 +      /* UNLOAD_METADATA_TABLETS */   {_,     _,        X,        X,      X,         X,          X},
 +      /* UNLOAD_ROOT_TABLET */        {_,     _,        _,        X,      X,         X,          X},
 +      /* STOP */                      {_,     _,        _,        _,      _,         X,          X}};
 +  //@formatter:on
 +  synchronized void setMasterState(MasterState newState) {
 +    if (state.equals(newState))
 +      return;
 +    if (!transitionOK[state.ordinal()][newState.ordinal()]) {
 +      log.error("Programmer error: master should not transition from " + state + " to " + newState);
 +    }
 +    MasterState oldState = state;
 +    state = newState;
 +    nextEvent.event("State changed from %s to %s", oldState, newState);
 +    if (newState == MasterState.STOP) {
 +      // Give the server a little time before shutdown so the client
 +      // thread requesting the stop can return
 +      SimpleTimer.getInstance().schedule(new Runnable() {
 +        @Override
 +        public void run() {
 +          // This frees the main thread and will cause the master to exit
 +          clientService.stop();
 +          Master.this.nextEvent.event("stopped event loop");
 +        }
 +
 +      }, 100l, 1000l);
 +    }
 +
 +    if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {
 +      upgradeZookeeper();
 +    }
 +
 +    if (oldState != newState && (newState == MasterState.NORMAL)) {
 +      upgradeMetadata();
 +    }
 +  }
 +
 +  private void moveRootTabletToRootTable(IZooReaderWriter zoo) throws Exception {
 +    String dirZPath = ZooUtil.getRoot(instance) + RootTable.ZROOT_TABLET_PATH;
 +
 +    if (!zoo.exists(dirZPath)) {
 +      Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
 +      if (fs.exists(oldPath)) {
 +        String newPath = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
 +        fs.mkdirs(new Path(newPath));
 +        if (!fs.rename(oldPath, new Path(newPath))) {
 +          throw new IOException("Failed to move root tablet from " + oldPath + " to " + newPath);
 +        }
 +
 +        log.info("Upgrade renamed " + oldPath + " to " + newPath);
 +      }
 +
 +      Path location = null;
 +
 +      for (String basePath : ServerConstants.getTablesDirs()) {
 +        Path path = new Path(basePath + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION);
 +        if (fs.exists(path)) {
 +          if (location != null) {
 +            throw new IllegalStateException("Root table at multiple locations " + location + " " + path);
 +          }
 +
 +          location = path;
 +        }
 +      }
 +
 +      if (location == null)
 +        throw new IllegalStateException("Failed to find root tablet");
 +
 +      log.info("Upgrade setting root table location in zookeeper " + location);
 +      zoo.putPersistentData(dirZPath, location.toString().getBytes(), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  private boolean haveUpgradedZooKeeper = false;
 +
 +  private void upgradeZookeeper() {
 +    // 1.5.1 and 1.6.0 both do some state checking after obtaining the zoolock for the
 +    // monitor and before starting up. It's not tied to the data version at all (and would
 +    // introduce unnecessary complexity to try to make the master do it), but be aware
 +    // that the master is not the only thing that may alter zookeeper before starting.
 +
 +    final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
 +    if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
 +      // This Master hasn't started Fate yet, so any outstanding transactions must be from before the upgrade.
 +      // Change to Guava's Verify once we use Guava 17.
 +      if (null != fate) {
 +        throw new IllegalStateException(
 +            "Access to Fate should not have been initialized prior to the Master transitioning to active. Please save all logs and file a bug.");
 +      }
 +      Accumulo.abortIfFateTransactions();
 +      try {
 +        log.info("Upgrading zookeeper");
 +
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        final String zooRoot = ZooUtil.getRoot(instance);
 +
 +        if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
 +          log.debug("Handling updates for version " + ServerConstants.TWO_DATA_VERSIONS_AGO);
 +
 +          log.debug("Cleaning out remnants of logger role.");
 +          zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
 +          zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
 +
 +          final byte[] zero = new byte[] {'0'};
 +          log.debug("Initializing recovery area.");
 +          zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
 +
 +          for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
 +            log.debug("Prepping table " + id + " for compaction cancellations.");
 +            zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
 +          }
 +        }
 +
 +        // create initial namespaces
 +        String namespaces = ZooUtil.getRoot(instance) + Constants.ZNAMESPACES;
 +        zoo.putPersistentData(namespaces, new byte[0], NodeExistsPolicy.SKIP);
 +        for (Pair<String,String> namespace : Iterables.concat(
 +            Collections.singleton(new Pair<String,String>(Namespaces.ACCUMULO_NAMESPACE, Namespaces.ACCUMULO_NAMESPACE_ID)),
 +            Collections.singleton(new Pair<String,String>(Namespaces.DEFAULT_NAMESPACE, Namespaces.DEFAULT_NAMESPACE_ID)))) {
 +          String ns = namespace.getFirst();
 +          String id = namespace.getSecond();
 +          log.debug("Upgrade creating namespace \"" + ns + "\" (ID: " + id + ")");
 +          if (!Namespaces.exists(instance, id))
 +            TableManager.prepareNewNamespaceState(instance.getInstanceID(), id, ns, NodeExistsPolicy.SKIP);
 +        }
 +
 +        // create root table
 +        log.debug("Upgrade creating table " + RootTable.NAME + " (ID: " + RootTable.ID + ")");
 +        TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE,
 +            NodeExistsPolicy.SKIP);
 +        Initialize.initMetadataConfig();
 +        // ensure root user can flush root table
 +        security.grantTablePermission(SystemCredentials.get().toThrift(instance), security.getRootUsername(), RootTable.ID, TablePermission.ALTER_TABLE,
 +            Namespaces.ACCUMULO_NAMESPACE_ID);
 +
 +        // put existing tables in the correct namespaces
 +        String tables = ZooUtil.getRoot(instance) + Constants.ZTABLES;
 +        for (String tableId : zoo.getChildren(tables)) {
 +          String targetNamespace = (MetadataTable.ID.equals(tableId) || RootTable.ID.equals(tableId)) ? Namespaces.ACCUMULO_NAMESPACE_ID
 +              : Namespaces.DEFAULT_NAMESPACE_ID;
-           log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), Constants.UTF8) + " (ID: "
++          log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), UTF_8) + " (ID: "
 +              + tableId + ") into namespace with ID " + targetNamespace);
-           zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++          zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(UTF_8), NodeExistsPolicy.SKIP);
 +        }
 +
 +        // rename metadata table
 +        log.debug("Upgrade renaming table " + MetadataTable.OLD_NAME + " (ID: " + MetadataTable.ID + ") to " + MetadataTable.NAME);
-         zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(Constants.UTF8),
++        zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8),
 +            NodeExistsPolicy.OVERWRITE);
 +
 +        moveRootTabletToRootTable(zoo);
 +
 +        // add system namespace permissions to existing users
 +        ZKPermHandler perm = new ZKPermHandler();
 +        perm.initialize(instance.getInstanceID(), true);
 +        String users = ZooUtil.getRoot(instance) + "/users";
 +        for (String user : zoo.getChildren(users)) {
 +          zoo.putPersistentData(users + "/" + user + "/Namespaces", new byte[0], NodeExistsPolicy.SKIP);
 +          perm.grantNamespacePermission(user, Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.READ);
 +        }
 +        perm.grantNamespacePermission("root", Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.ALTER_TABLE);
 +        haveUpgradedZooKeeper = true;
 +      } catch (Exception ex) {
 +        log.fatal("Error performing upgrade", ex);
 +        System.exit(1);
 +      }
 +    }
 +  }
 +
 +  private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
 +  private final CountDownLatch waitForMetadataUpgrade = new CountDownLatch(1);
 +
 +  private final ServerConfiguration serverConfig;
 +
 +  private void upgradeMetadata() {
 +    // we make sure we're only doing the rest of this method once so that we can signal to other threads that an upgrade wasn't needed.
 +    if (upgradeMetadataRunning.compareAndSet(false, true)) {
 +      final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
 +      if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
 +        // sanity check that we passed the Fate verification prior to ZooKeeper upgrade, and that Fate still hasn't been started.
 +        // Change both to use Guava's Verify once we use Guava 17.
 +        if (!haveUpgradedZooKeeper) {
 +          throw new IllegalStateException(
 +              "We should only attempt to upgrade Accumulo's metadata table if we've already upgraded ZooKeeper. Please save all logs and file a bug.");
 +        }
 +        if (null != fate) {
 +          throw new IllegalStateException(
 +              "Access to Fate should not have been initialized prior to the Master finishing upgrades. Please save all logs and file a bug.");
 +        }
 +        Runnable upgradeTask = new Runnable() {
 +          @Override
 +          public void run() {
 +            try {
 +              log.info("Starting to upgrade metadata table.");
 +              if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
 +                log.info("Updating Delete Markers in metadata table for version 1.4");
 +                MetadataTableUtil.moveMetaDeleteMarkersFrom14(instance, SystemCredentials.get());
 +              } else {
 +                log.info("Updating Delete Markers in metadata table.");
 +                MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
 +              }
 +              log.info("Updating persistent data version.");
 +              Accumulo.updateAccumuloVersion(fs, accumuloPersistentVersion);
 +              log.info("Upgrade complete");
 +              waitForMetadataUpgrade.countDown();
 +            } catch (Exception ex) {
 +              log.fatal("Error performing upgrade", ex);
 +              System.exit(1);
 +            }
 +
 +          }
 +        };
 +
 +        // need to run this in a separate thread because a lock is held that prevents metadata tablets from being assigned and this task writes to the
 +        // metadata table
 +        new Thread(upgradeTask).start();
 +      } else {
 +        waitForMetadataUpgrade.countDown();
 +      }
 +    }
 +  }
 +
 +  private int assignedOrHosted(Text tableId) {
 +    int result = 0;
 +    for (TabletGroupWatcher watcher : watchers) {
 +      TableCounts count = watcher.getStats(tableId);
 +      result += count.hosted() + count.assigned();
 +    }
 +    return result;
 +  }
 +
 +  private int totalAssignedOrHosted() {
 +    int result = 0;
 +    for (TabletGroupWatcher watcher : watchers) {
 +      for (TableCounts counts : watcher.getStats().values()) {
 +        result += counts.assigned() + counts.hosted();
 +      }
 +    }
 +    return result;
 +  }
 +
 +  private int nonMetaDataTabletsAssignedOrHosted() {
 +    return totalAssignedOrHosted() - assignedOrHosted(new Text(MetadataTable.ID)) - assignedOrHosted(new Text(RootTable.ID));
 +  }
 +
 +  private int notHosted() {
 +    int result = 0;
 +    for (TabletGroupWatcher watcher : watchers) {
 +      for (TableCounts counts : watcher.getStats().values()) {
 +        result += counts.assigned() + counts.assignedToDeadServers();
 +      }
 +    }
 +    return result;
 +  }
 +
 +  // The number of unassigned tablets that should be assigned: displayed on the monitor page
 +  int displayUnassigned() {
 +    int result = 0;
 +    switch (getMasterState()) {
 +      case NORMAL:
 +        // Count offline tablets for online tables
 +        for (TabletGroupWatcher watcher : watchers) {
 +          TableManager manager = TableManager.getInstance();
 +          for (Entry<Text,TableCounts> entry : watcher.getStats().entrySet()) {
 +            Text tableId = entry.getKey();
 +            TableCounts counts = entry.getValue();
 +            TableState tableState = manager.getTableState(tableId.toString());
 +            if (tableState != null && tableState.equals(TableState.ONLINE)) {
 +              result += counts.unassigned() + counts.assignedToDeadServers() + counts.assigned();
 +            }
 +          }
 +        }
 +        break;
 +      case SAFE_MODE:
 +        // Count offline tablets for the metadata table
 +        for (TabletGroupWatcher watcher : watchers) {
 +          result += watcher.getStats(METADATA_TABLE_ID).unassigned();
 +        }
 +        break;
 +      case UNLOAD_METADATA_TABLETS:
 +      case UNLOAD_ROOT_TABLET:
 +        for (TabletGroupWatcher watcher : watchers) {
 +          result += watcher.getStats(METADATA_TABLE_ID).unassigned();
 +        }
 +        break;
 +      default:
 +        break;
 +    }
 +    return result;
 +  }
 +
 +  public void mustBeOnline(final String tableId) throws ThriftTableOperationException {
 +    Tables.clearCache(instance);
 +    if (!Tables.getTableState(instance, tableId).equals(TableState.ONLINE))
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.MERGE, TableOperationExceptionType.OFFLINE, "table is not online");
 +  }
 +
 +  public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
 +    return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
 +  }
 +
 +  private Master(ServerConfiguration config, VolumeManager fs, String hostname) throws IOException {
 +    this.serverConfig = config;
 +    this.instance = config.getInstance();
 +    this.fs = fs;
 +    this.hostname = hostname;
 +
 +    AccumuloConfiguration aconf = serverConfig.getConfiguration();
 +
 +    log.info("Version " + Constants.VERSION);
 +    log.info("Instance " + instance.getInstanceID());
 +    ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
 +    security = AuditedSecurityOperation.getInstance();
 +    tserverSet = new LiveTServerSet(instance, config.getConfiguration(), this);
 +    this.tabletBalancer = aconf.instantiateClassProperty(Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
 +    this.tabletBalancer.init(serverConfig);
 +
 +    try {
 +      AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() {
 +        @Override
 +        public Iterator<Entry<String,String>> iterator() {
 +          return getSystemConfiguration().iterator();
 +        }
 +      }));
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +
 +  }
 +
 +  public TServerConnection getConnection(TServerInstance server) {
 +    return tserverSet.getConnection(server);
 +  }
 +
 +  public MergeInfo getMergeInfo(Text tableId) {
 +    synchronized (mergeLock) {
 +      try {
 +        String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
 +        if (!ZooReaderWriter.getInstance().exists(path))
 +          return new MergeInfo();
 +        byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat());
 +        DataInputBuffer in = new DataInputBuffer();
 +        in.reset(data, data.length);
 +        MergeInfo info = new MergeInfo();
 +        info.readFields(in);
 +        return info;
 +      } catch (KeeperException.NoNodeException ex) {
 +        log.info("Error reading merge state, it probably just finished");
 +        return new MergeInfo();
 +      } catch (Exception ex) {
 +        log.warn("Unexpected error reading merge state", ex);
 +        return new MergeInfo();
 +      }
 +    }
 +  }
 +
 +  public void setMergeState(MergeInfo info, MergeState state) throws IOException, KeeperException, InterruptedException {
 +    synchronized (mergeLock) {
 +      String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + info.getExtent().getTableId().toString() + "/merge";
 +      info.setState(state);
 +      if (state.equals(MergeState.NONE)) {
 +        ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
 +      } else {
 +        DataOutputBuffer out = new DataOutputBuffer();
 +        try {
 +          info.write(out);
 +        } catch (IOException ex) {
 +          throw new RuntimeException("Unlikely", ex);
 +        }
 +        ZooReaderWriter.getInstance().putPersistentData(path, out.getData(),
 +            state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE);
 +      }
 +      mergeLock.notifyAll();
 +    }
 +    nextEvent.event("Merge state of %s set to %s", info.getExtent(), state);
 +  }
 +
 +  public void clearMergeState(Text tableId) throws IOException, KeeperException, InterruptedException {
 +    synchronized (mergeLock) {
 +      String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
 +      ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
 +      mergeLock.notifyAll();
 +    }
 +    nextEvent.event("Merge state of %s cleared", tableId);
 +  }
 +
 +  void setMasterGoalState(MasterGoalState state) {
 +    try {
 +      ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, state.name().getBytes(),
 +          NodeExistsPolicy.OVERWRITE);
 +    } catch (Exception ex) {
 +      log.error("Unable to set master goal state in zookeeper");
 +    }
 +  }
 +
 +  MasterGoalState getMasterGoalState() {
 +    while (true)
 +      try {
 +        byte[] data = ZooReaderWriter.getInstance().getData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, null);
 +        return MasterGoalState.valueOf(new String(data));
 +      } catch (Exception e) {
 +        log.error("Problem getting real goal state: " + e);
 +        UtilWaitThread.sleep(1000);
 +      }
 +  }
 +
 +  public boolean hasCycled(long time) {
 +    for (TabletGroupWatcher watcher : watchers) {
 +      if (watcher.stats.lastScanFinished() < time)
 +        return false;
 +    }
 +
 +    return true;
 +  }
 +
 +  public void clearMigrations(String tableId) {
 +    synchronized (migrations) {
 +      Iterator<KeyExtent> iterator = migrations.keySet().iterator();
 +      while (iterator.hasNext()) {
 +        KeyExtent extent = iterator.next();
 +        if (extent.getTableId().toString().equals(tableId)) {
 +          iterator.remove();
 +        }
 +      }
 +    }
 +  }
 +
 +  static enum TabletGoalState {
 +    HOSTED, UNASSIGNED, DELETED
 +  };
 +
 +  TabletGoalState getSystemGoalState(TabletLocationState tls) {
 +    switch (getMasterState()) {
 +      case NORMAL:
 +        return TabletGoalState.HOSTED;
 +      case HAVE_LOCK: // fall-through intended
 +      case INITIAL: // fall-through intended
 +      case SAFE_MODE:
 +        if (tls.extent.isMeta())
 +          return TabletGoalState.HOSTED;
 +        return TabletGoalState.UNASSIGNED;
 +      case UNLOAD_METADATA_TABLETS:
 +        if (tls.extent.isRootTablet())
 +          return TabletGoalState.HOSTED;
 +        return TabletGoalState.UNASSIGNED;
 +      case UNLOAD_ROOT_TABLET:
 +        return TabletGoalState.UNASSIGNED;
 +      case STOP:
 +        return TabletGoalState.UNASSIGNED;
 +      default:
 +        throw new IllegalStateException("Unknown Master State");
 +    }
 +  }
 +
 +  TabletGoalState getTableGoalState(KeyExtent extent) {
 +    TableState tableState = TableManager.getInstance().getTableState(extent.getTableId().toString());
 +    if (tableState == null)
 +      return TabletGoalState.DELETED;
 +    switch (tableState) {
 +      case DELETING:
 +        return TabletGoalState.DELETED;
 +      case OFFLINE:
 +      case NEW:
 +        return TabletGoalState.UNASSIGNED;
 +      default:
 +        return TabletGoalState.HOSTED;
 +    }
 +  }
 +
 +  TabletGoalState getGoalState(TabletLocationState tls, MergeInfo mergeInfo) {
 +    KeyExtent extent = tls.extent;
 +    // Shutting down?
 +    TabletGoalState state = getSystemGoalState(tls);
 +    if (state == TabletGoalState.HOSTED) {
 +      if (tls.current != null && serversToShutdown.contains(tls.current)) {
 +        return TabletGoalState.UNASSIGNED;
 +      }
 +      // Handle merge transitions
 +      if (mergeInfo.getExtent() != null) {
 +        log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent));
 +        if (mergeInfo.overlaps(extent)) {
 +          switch (mergeInfo.getState()) {
 +            case NONE:
 +            case COMPLETE:
 +              break;
 +            case STARTED:
 +            case SPLITTING:
 +              return TabletGoalState.HOSTED;
 +            case WAITING_FOR_CHOPPED:
 +              if (tls.getState(onlineTabletServers()).equals(TabletState.HOSTED)) {
 +                if (tls.chopped)
 +                  return TabletGoalState.UNASSIGNED;
 +              } else {
 +                if (tls.chopped && tls.walogs.isEmpty())
 +                  return TabletGoalState.UNASSIGNED;
 +              }
 +
 +              return TabletGoalState.HOSTED;
 +            case WAITING_FOR_OFFLINE:
 +            case MERGING:
 +              return TabletGoalState.UNASSIGNED;
 +          }
 +        }
 +      }
 +
 +      // taking table offline?
 +      state = getTableGoalState(extent);
 +      if (state == TabletGoalState.HOSTED) {
 +        // Maybe this tablet needs to be migrated
 +        TServerInstance dest = migrations.get(extent);
 +        if (dest != null && tls.current != null && !dest.equals(tls.current)) {
 +          return TabletGoalState.UNASSIGNED;
 +        }
 +      }
 +    }
 +    return state;
 +  }
 +
 +  private class MigrationCleanupThread extends Daemon {
 +
 +    @Override
 +    public void run() {
 +      setName("Migration Cleanup Thread");
 +      while (stillMaster()) {
 +        if (!migrations.isEmpty()) {
 +          try {
 +            cleanupOfflineMigrations();
 +            cleanupNonexistentMigrations(getConnector());
 +          } catch (Exception ex) {
 +            log.error("Error cleaning up migrations", ex);
 +          }
 +        }
 +        UtilWaitThread.sleep(TIME_BETWEEN_MIGRATION_CLEANUPS);
 +      }
 +    }
 +
 +    /**
 +     * If a migrating tablet splits, and the tablet dies before sending the master a message, the migration will refer to a non-existing tablet, so it can never
 +     * complete. Periodically scan the metadata table and remove any migrating tablets that no longer exist.
 +     */
 +    private void cleanupNonexistentMigrations(final Connector connector) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +      Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +      TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      Set<KeyExtent> found = new HashSet<KeyExtent>();
 +      for (Entry<Key,Value> entry : scanner) {
 +        KeyExtent extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
 +        if (migrations.containsKey(extent)) {
 +          found.add(extent);
 +        }
 +      }
 +      migrations.keySet().retainAll(found);
 +    }
 +
 +    /**
 +     * If migrating a tablet for a table that is offline, the migration can never succeed because no tablet server will load the tablet. check for offline
 +     * tables and remove their migrations.
 +     */
 +    private void cleanupOfflineMigrations() {
 +      TableManager manager = TableManager.getInstance();
 +      for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
 +        TableState state = manager.getTableState(tableId);
 +        if (TableState.OFFLINE == state) {
 +          clearMigrations(tableId);
 +        }
 +      }
 +    }
 +  }
 +
 +  private class StatusThread extends Daemon {
 +
 +    @Override
 +    public void run() {
 +      setName("Status Thread");
 +      EventCoordinator.Listener eventListener = nextEvent.getListener();
 +      while (stillMaster()) {
 +        long wait = DEFAULT_WAIT_FOR_WATCHER;
 +        try {
 +          switch (getMasterGoalState()) {
 +            case NORMAL:
 +              setMasterState(MasterState.NORMAL);
 +              break;
 +            case SAFE_MODE:
 +              if (getMasterState() == MasterState.NORMAL) {
 +                setMasterState(MasterState.SAFE_MODE);
 +              }
 +              if (getMasterState() == MasterState.HAVE_LOCK) {
 +                setMasterState(MasterState.SAFE_MODE);
 +              }
 +              break;
 +            case CLEAN_STOP:
 +              switch (getMasterState()) {
 +                case NORMAL:
 +                  setMasterState(MasterState.SAFE_MODE);
 +                  break;
 +                case SAFE_MODE: {
 +                  int count = nonMetaDataTabletsAssignedOrHosted();
 +                  log.debug(String.format("There are %d non-metadata tablets assigned or hosted", count));
 +                  if (count == 0)
 +                    setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
 +                }
 +                  break;
 +                case UNLOAD_METADATA_TABLETS: {
 +                  int count = assignedOrHosted(METADATA_TABLE_ID);
 +                  log.debug(String.format("There are %d metadata tablets assigned or hosted", count));
 +                  if (count == 0)
 +                    setMasterState(MasterState.UNLOAD_ROOT_TABLET);
 +                }
 +                  break;
 +                case UNLOAD_ROOT_TABLET: {
 +                  int count = assignedOrHosted(METADATA_TABLE_ID);
 +                  if (count > 0) {
 +                    log.debug(String.format("%d metadata tablets online", count));
 +                    setMasterState(MasterState.UNLOAD_ROOT_TABLET);
 +                  }
 +                  int root_count = assignedOrHosted(ROOT_TABLE_ID);
 +                  if (root_count > 0)
 +                    log.debug("The root tablet is still assigned or hosted");
 +                  if (count + root_count == 0) {
 +                    Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
 +                    log.debug("stopping " + currentServers.size() + " tablet servers");
 +                    for (TServerInstance server : currentServers) {
 +                      try {
 +                        serversToShutdown.add(server);
 +                        tserverSet.getConnection(server).fastHalt(masterLock);
 +                      } catch (TException e) {
 +                        // its probably down, and we don't care
 +                      } finally {
 +                        tserverSet.remove(server);
 +                      }
 +                    }
 +                    if (currentServers.size() == 0)
 +                      setMasterState(MasterState.STOP);
 +                  }
 +                }
 +                  break;
 +                default:
 +                  break;
 +              }
 +          }
 +          wait = updateStatus();
 +          eventListener.waitForEvents(wait);
 +        } catch (Throwable t) {
 +          log.error("Error balancing tablets", t);
 +          UtilWaitThread.sleep(WAIT_BETWEEN_ERRORS);
 +        }
 +      }
 +    }
 +
 +    private long updateStatus() throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +      tserverStatus = Collections.synchronizedSortedMap(gatherTableInformation());
 +      checkForHeldServer(tserverStatus);
 +
 +      if (!badServers.isEmpty()) {
 +        log.debug("not balancing because the balance information is out-of-date " + badServers.keySet());
 +      } else if (notHosted() > 0) {
 +        log.debug("not balancing because there are unhosted tablets: " + notHosted());
 +      } else if (getMasterGoalState() == MasterGoalState.CLEAN_STOP) {
 +        log.debug("not balancing because the master is attempting to stop cleanly");
 +      } else if (!serversToShutdown.isEmpty()) {
 +        log.debug("not balancing while shutting down servers " + serversToShutdown);
 +      } else {
 +        return balanceTablets();
 +      }
 +      return DEFAULT_WAIT_FOR_WATCHER;
 +    }
 +
 +    private void checkForHeldServer(SortedMap<TServerInstance,TabletServerStatus> tserverStatus) {
 +      TServerInstance instance = null;
 +      int crazyHoldTime = 0;
 +      int someHoldTime = 0;
 +      final long maxWait = getSystemConfiguration().getTimeInMillis(Property.TSERV_HOLD_TIME_SUICIDE);
 +      for (Entry<TServerInstance,TabletServerStatus> entry : tserverStatus.entrySet()) {
 +        if (entry.getValue().getHoldTime() > 0) {
 +          someHoldTime++;
 +          if (entry.getValue().getHoldTime() > maxWait) {
 +            instance = entry.getKey();
 +            crazyHoldTime++;
 +          }
 +        }
 +      }
 +      if (crazyHoldTime == 1 && someHoldTime == 1 && tserverStatus.size() > 1) {
 +        log.warn("Tablet server " + instance + " exceeded maximum hold time: attempting to kill it");
 +        try {
 +          TServerConnection connection = tserverSet.getConnection(instance);
 +          if (connection != null)
 +            connection.fastHalt(masterLock);
 +        } catch (TException e) {
 +          log.error(e, e);
 +        }
 +        tserverSet.remove(instance);
 +      }
 +    }
 +
 +    private long balanceTablets() {
 +      List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
 +      Set<KeyExtent> migrationsCopy = new HashSet<KeyExtent>();
 +      synchronized (migrations) {
 +        migrationsCopy.addAll(migrations.keySet());
 +      }
 +      long wait = tabletBalancer.balance(Collections.unmodifiableSortedMap(tserverStatus), Collections.unmodifiableSet(migrationsCopy), migrationsOut);
 +
 +      for (TabletMigration m : TabletBalancer.checkMigrationSanity(tserverStatus.keySet(), migrationsOut)) {
 +        if (migrations.containsKey(m.tablet)) {
 +          log.warn("balancer requested migration more than once, skipping " + m);
 +          continue;
 +        }
 +        migrations.put(m.tablet, m.newServer);
 +        log.debug("migration " + m);
 +      }
 +      if (migrationsOut.size() > 0) {
 +        nextEvent.event("Migrating %d more tablets, %d total", migrationsOut.size(), migrations.size());
 +      }
 +      return wait;
 +    }
 +
 +  }
 +
 +  private SortedMap<TServerInstance,TabletServerStatus> gatherTableInformation() {
 +    long start = System.currentTimeMillis();
 +    SortedMap<TServerInstance,TabletServerStatus> result = new TreeMap<TServerInstance,TabletServerStatus>();
 +    Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
 +    for (TServerInstance server : currentServers) {
 +      try {
 +        Thread t = Thread.currentThread();
 +        String oldName = t.getName();
 +        try {
 +          t.setName("Getting status from " + server);
 +          TServerConnection connection = tserverSet.getConnection(server);
 +          if (connection == null)
 +            throw new IOException("No connection to " + server);
 +          TabletServerStatus status = connection.getTableMap(false);
 +          result.put(server, status);
 +        } finally {
 +          t.setName(oldName);
 +        }
 +      } catch (Exception ex) {
 +        log.error("unable to get tablet server status " + server + " " + ex.toString());
 +        log.debug("unable to get tablet server status " + server, ex);
 +        if (badServers.get(server).incrementAndGet() > MAX_BAD_STATUS_COUNT) {
 +          log.warn("attempting to stop " + server);
 +          try {
 +            TServerConnection connection = tserverSet.getConnection(server);
 +            if (connection != null)
 +              connection.halt(masterLock);
 +          } catch (TTransportException e) {
 +            // ignore: it's probably down
 +          } catch (Exception e) {
 +            log.info("error talking to troublesome tablet server ", e);
 +          }
 +          badServers.remove(server);
 +          tserverSet.remove(server);
 +        }
 +      }
 +    }
 +    synchronized (badServers) {
 +      badServers.keySet().retainAll(currentServers);
 +      badServers.keySet().removeAll(result.keySet());
 +    }
 +    log.debug(String.format("Finished gathering information from %d servers in %.2f seconds", result.size(), (System.currentTimeMillis() - start) / 1000.));
 +    return result;
 +  }
 +
 +  public void run() throws IOException, InterruptedException, KeeperException {
 +    final String zroot = ZooUtil.getRoot(instance);
 +
 +    getMasterLock(zroot + Constants.ZMASTER_LOCK);
 +
 +    recoveryManager = new RecoveryManager(this);
 +
 +    TableManager.getInstance().addObserver(this);
 +
 +    StatusThread statusThread = new StatusThread();
 +    statusThread.start();
 +
 +    MigrationCleanupThread migrationCleanupThread = new MigrationCleanupThread();
 +    migrationCleanupThread.start();
 +
 +    tserverSet.startListeningForTabletServerChanges();
 +
 +    ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, new Watcher() {
 +      @Override
 +      public void process(WatchedEvent event) {
 +        nextEvent.event("Noticed recovery changes", event.getType());
 +        try {
 +          // watcher only fires once, add it back
 +          ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, this);
 +        } catch (Exception e) {
 +          log.error("Failed to add log recovery watcher back", e);
 +        }
 +      }
 +    });
 +
 +    Credentials systemCreds = SystemCredentials.get();
 +    watchers.add(new TabletGroupWatcher(this, new MetaDataStateStore(instance, systemCreds, this), null));
 +    watchers.add(new TabletGroupWatcher(this, new RootTabletStateStore(instance, systemCreds, this), watchers.get(0)));
 +    watchers.add(new TabletGroupWatcher(this, new ZooTabletStateStore(new ZooStore(zroot)), watchers.get(1)));
 +    for (TabletGroupWatcher watcher : watchers) {
 +      watcher.start();
 +    }
 +
 +    // Once we are sure the upgrade is complete, we can safely allow fate use.
 +    waitForMetadataUpgrade.await();
 +
 +    try {
 +      final AgeOffStore<Master> store = new AgeOffStore<Master>(new org.apache.accumulo.fate.ZooStore<Master>(ZooUtil.getRoot(instance) + Constants.ZFATE,
 +          ZooReaderWriter.getInstance()), 1000 * 60 * 60 * 8);
 +
 +      int threads = this.getConfiguration().getConfiguration().getCount(Property.MASTER_FATE_THREADPOOL_SIZE);
 +
 +      fate = new Fate<Master>(this, store, threads);
 +
 +      SimpleTimer.getInstance().schedule(new Runnable() {
 +
 +        @Override
 +        public void run() {
 +          store.ageOff();
 +        }
 +      }, 63000, 63000);
 +    } catch (KeeperException e) {
 +      throw new IOException(e);
 +    } catch (InterruptedException e) {
 +      throw new IOException(e);
 +    }
 +
 +    Processor<Iface> processor = new Processor<Iface>(RpcWrapper.service(new MasterClientServiceHandler(this)));
 +    ServerAddress sa = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_CLIENTPORT, processor, "Master",
 +        "Master Client Service Handler", null, Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
 +    clientService = sa.server;
 +    String address = sa.address.toString();
 +    log.info("Setting master lock data to " + address);
 +    masterLock.replaceLockData(address.getBytes());
 +
 +    while (!clientService.isServing()) {
 +      UtilWaitThread.sleep(100);
 +    }
 +    while (clientService.isServing()) {
 +      UtilWaitThread.sleep(500);
 +    }
 +    log.info("Shutting down fate.");
 +    fate.shutdown();
 +
 +    final long deadline = System.currentTimeMillis() + MAX_CLEANUP_WAIT_TIME;
 +    statusThread.join(remaining(deadline));
 +
 +    // quit, even if the tablet servers somehow jam up and the watchers
 +    // don't stop
 +    for (TabletGroupWatcher watcher : watchers) {
 +      watcher.join(remaining(deadline));
 +    }
 +    log.info("exiting");
 +  }
 +
 +  private long remaining(long deadline) {
 +    return Math.max(1, deadline - System.currentTimeMillis());
 +  }
 +
 +  public ZooLock getMasterLock() {
 +    return masterLock;
 +  }
 +
 +  private static class MasterLockWatcher implements ZooLock.AsyncLockWatcher {
 +
 +    boolean acquiredLock = false;
 +    boolean failedToAcquireLock = false;
 +
 +    @Override
 +    public void lostLock(LockLossReason reason) {
 +      Halt.halt("Master lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
 +    }
 +
 +    @Override
 +    public void unableToMonitorLockNode(final Throwable e) {
 +      Halt.halt(-1, new Runnable() {
 +        @Override
 +        public void run() {
 +          log.fatal("No longer able to monitor master lock node", e);
 +        }
 +      });
 +
 +    }
 +
 +    @Override
 +    public synchronized void acquiredLock() {
 +      log.debug("Acquired master lock");
 +
 +      if (acquiredLock || failedToAcquireLock) {
 +        Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      acquiredLock = true;
 +      notifyAll();
 +    }
 +
 +    @Override
 +    public synchronized void failedToAcquireLock(Exception e) {
 +      log.warn("Failed to get master lock " + e);
 +
 +      if (acquiredLock) {
 +        Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      failedToAcquireLock = true;
 +      notifyAll();
 +    }
 +
 +    public synchronized void waitForChange() {
 +      while (!acquiredLock && !failedToAcquireLock) {
 +        try {
 +          wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +  }
 +
 +  private void getMasterLock(final String zMasterLoc) throws KeeperException, InterruptedException {
 +    log.info("trying to get master lock");
 +
 +    final String masterClientAddress = hostname + ":" + getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
 +
 +    while (true) {
 +
 +      MasterLockWatcher masterLockWatcher = new MasterLockWatcher();
 +      masterLock = new ZooLock(zMasterLoc);
 +      masterLock.lockAsync(masterLockWatcher, masterClientAddress.getBytes());
 +
 +      masterLockWatcher.waitForChange();
 +
 +      if (masterLockWatcher.acquiredLock) {
 +        break;
 +      }
 +
 +      if (!masterLockWatcher.failedToAcquireLock) {
 +        throw new IllegalStateException("master lock in unknown state");
 +      }
 +
 +      masterLock.tryToCancelAsyncLockOrUnlock();
 +
 +      UtilWaitThread.sleep(TIME_TO_WAIT_BETWEEN_LOCK_CHECKS);
 +    }
 +
 +    setMasterState(MasterState.HAVE_LOCK);
 +  }
 +
 +  public static void main(String[] args) throws Exception {
 +    try {
 +      SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +
 +      ServerOpts opts = new ServerOpts();
 +      final String app = "master";
 +      opts.parseArgs(app, args);
 +      String hostname = opts.getAddress();
 +      Accumulo.setupLogging(app);
 +      Instance instance = HdfsZooInstance.getInstance();
 +      ServerConfiguration conf = new ServerConfiguration(instance);
 +      VolumeManager fs = VolumeManagerImpl.get();
 +      Accumulo.init(fs, conf, app);
 +      Master master = new Master(conf, fs, hostname);
 +      Accumulo.enableTracing(hostname, app);
 +      master.run();
 +    } catch (Exception ex) {
 +      log.error("Unexpected exception, exiting", ex);
 +      System.exit(1);
 +    }
 +  }
 +
 +  @Override
 +  public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
 +    DeadServerList obit = new DeadServerList(ZooUtil.getRoot(instance) + Constants.ZDEADTSERVERS);
 +    if (added.size() > 0) {
 +      log.info("New servers: " + added);
 +      for (TServerInstance up : added)
 +        obit.delete(up.hostPort());
 +    }
 +    for (TServerInstance dead : deleted) {
 +      String cause = "unexpected failure";
 +      if (serversToShutdown.contains(dead))
 +        cause = "clean shutdown"; // maybe an incorrect assumption
 +      if (!getMasterGoalState().equals(MasterGoalState.CLEAN_STOP))
 +        obit.post(dead.hostPort(), cause);
 +    }
 +
 +    Set<TServerInstance> unexpected = new HashSet<TServerInstance>(deleted);
 +    unexpected.removeAll(this.serversToShutdown);
 +    if (unexpected.size() > 0) {
 +      if (stillMaster() && !getMasterGoalState().equals(MasterGoalState.CLEAN_STOP)) {
 +        log.warn("Lost servers " + unexpected);
 +      }
 +    }
 +    serversToShutdown.removeAll(deleted);
 +    badServers.keySet().removeAll(deleted);
 +    // clear out any bad server with the same host/port as a new server
 +    synchronized (badServers) {
 +      cleanListByHostAndPort(badServers.keySet(), deleted, added);
 +    }
 +    synchronized (serversToShutdown) {
 +      cleanListByHostAndPort(serversToShutdown, deleted, added);
 +    }
 +
 +    synchronized (migrations) {
 +      Iterator<Entry<KeyExtent,TServerInstance>> iter = migrations.entrySet().iterator();
 +      while (iter.hasNext()) {
 +        Entry<KeyExtent,TServerInstance> entry = iter.next();
 +        if (deleted.contains(entry.getValue())) {
 +          log.info("Canceling migration of " + entry.getKey() + " to " + entry.getValue());
 +          iter.remove();
 +        }
 +      }
 +    }
 +    nextEvent.event("There are now %d tablet servers", current.size());
 +  }
 +
 +  private static void cleanListByHostAndPort(Collection<TServerInstance> badServers, Set<TServerInstance> deleted, Set<TServerInstance> added) {
 +    Iterator<TServerInstance> badIter = badServers.iterator();
 +    while (badIter.hasNext()) {
 +      TServerInstance bad = badIter.next();
 +      for (TServerInstance add : added) {
 +        if (bad.hostPort().equals(add.hostPort())) {
 +          badIter.remove();
 +          break;
 +        }
 +      }
 +      for (TServerInstance del : deleted) {
 +        if (bad.hostPort().equals(del.hostPort())) {
 +          badIter.remove();
 +          break;
 +        }
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void stateChanged(String tableId, TableState state) {
 +    nextEvent.event("Table state in zookeeper changed for %s to %s", tableId, state);
 +    if (TableState.OFFLINE == state) {
 +      clearMigrations(tableId);
 +    }
 +  }
 +
 +  @Override
 +  public void initialize(Map<String,TableState> tableIdToStateMap) {}
 +
 +  @Override
 +  public void sessionExpired() {}
 +
 +  @Override
 +  public Set<String> onlineTables() {
 +    Set<String> result = new HashSet<String>();
 +    if (getMasterState() != MasterState.NORMAL) {
 +      if (getMasterState() != MasterState.UNLOAD_METADATA_TABLETS)
 +        result.add(MetadataTable.ID);
 +      if (getMasterState() != MasterState.UNLOAD_ROOT_TABLET)
 +        result.add(RootTable.ID);
 +      return result;
 +    }
 +    TableManager manager = TableManager.getInstance();
 +
 +    for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
 +      TableState state = manager.getTableState(tableId);
 +      if (state != null) {
 +        if (state == TableState.ONLINE)
 +          result.add(tableId);
 +      }
 +    }
 +    return result;
 +  }
 +
 +  @Override
 +  public Set<TServerInstance> onlineTabletServers() {
 +    return tserverSet.getCurrentServers();
 +  }
 +
 +  @Override
 +  public Collection<MergeInfo> merges() {
 +    List<MergeInfo> result = new ArrayList<MergeInfo>();
 +    for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
 +      result.add(getMergeInfo(new Text(tableId)));
 +    }
 +    return result;
 +  }
 +
 +  // recovers state from the persistent transaction to shutdown a server
 +  public void shutdownTServer(TServerInstance server) {
 +    nextEvent.event("Tablet Server shutdown requested for %s", server);
 +    serversToShutdown.add(server);
 +  }
 +
 +  public EventCoordinator getEventCoordinator() {
 +    return nextEvent;
 +  }
 +
 +  public Instance getInstance() {
 +    return this.instance;
 +  }
 +
 +  public AccumuloConfiguration getSystemConfiguration() {
 +    return serverConfig.getConfiguration();
 +  }
 +
 +  public ServerConfiguration getConfiguration() {
 +    return serverConfig;
 +  }
 +
 +  public VolumeManager getFileSystem() {
 +    return this.fs;
 +  }
 +
 +  public void assignedTablet(KeyExtent extent) {
 +    if (extent.isMeta()) {
 +      if (getMasterState().equals(MasterState.UNLOAD_ROOT_TABLET)) {
 +        setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
 +      }
 +    }
 +    if (extent.isRootTablet()) {
 +      // probably too late, but try anyhow
 +      if (getMasterState().equals(MasterState.STOP)) {
 +        setMasterState(MasterState.UNLOAD_ROOT_TABLET);
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index 0c2e1f0,0000000..5e478cb
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@@ -1,190 -1,0 +1,192 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.recovery;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.ScheduledExecutorService;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.util.NamingThreadFactory;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.log.SortedLogState;
 +import org.apache.accumulo.server.master.recovery.HadoopLogCloser;
 +import org.apache.accumulo.server.master.recovery.LogCloser;
 +import org.apache.accumulo.server.master.recovery.RecoveryPath;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class RecoveryManager {
 +
 +  private static Logger log = Logger.getLogger(RecoveryManager.class);
 +
 +  private Map<String,Long> recoveryDelay = new HashMap<String,Long>();
 +  private Set<String> closeTasksQueued = new HashSet<String>();
 +  private Set<String> sortsQueued = new HashSet<String>();
 +  private ScheduledExecutorService executor;
 +  private Master master;
 +  private ZooCache zooCache;
 +
 +  public RecoveryManager(Master master) {
 +    this.master = master;
 +    executor = Executors.newScheduledThreadPool(4, new NamingThreadFactory("Walog sort starter "));
 +    zooCache = new ZooCache();
 +    try {
 +      List<String> workIDs = new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).getWorkQueued();
 +      sortsQueued.addAll(workIDs);
 +    } catch (Exception e) {
 +      log.warn(e, e);
 +    }
 +  }
 +
 +  private class LogSortTask implements Runnable {
 +    private String source;
 +    private String destination;
 +    private String sortId;
 +    private LogCloser closer;
 +
 +    public LogSortTask(LogCloser closer, String source, String destination, String sortId) {
 +      this.closer = closer;
 +      this.source = source;
 +      this.destination = destination;
 +      this.sortId = sortId;
 +    }
 +
 +    @Override
 +    public void run() {
 +      boolean rescheduled = false;
 +      try {
 +
 +        long time = closer.close(master.getConfiguration().getConfiguration(), master.getFileSystem(), new Path(source));
 +
 +        if (time > 0) {
 +          executor.schedule(this, time, TimeUnit.MILLISECONDS);
 +          rescheduled = true;
 +        } else {
 +          initiateSort(sortId, source, destination);
 +        }
 +      } catch (FileNotFoundException e) {
 +        log.debug("Unable to initate log sort for " + source + ": " + e);
 +      } catch (Exception e) {
 +        log.warn("Failed to initiate log sort " + source, e);
 +      } finally {
 +        if (!rescheduled) {
 +          synchronized (RecoveryManager.this) {
 +            closeTasksQueued.remove(sortId);
 +          }
 +        }
 +      }
 +    }
 +
 +  }
 +
 +  private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
 +    String work = source + "|" + destination;
-     new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(Constants.UTF8));
++    new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(UTF_8));
 +
 +    synchronized (this) {
 +      sortsQueued.add(sortId);
 +    }
 +
 +    final String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId;
 +    log.info("Created zookeeper entry " + path + " with data " + work);
 +  }
 +
 +  public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>> walogs) throws IOException {
 +    boolean recoveryNeeded = false;
 +
 +    for (Collection<String> logs : walogs) {
 +      for (String walog : logs) {
 +
 +        String switchedWalog = VolumeUtil.switchVolume(walog, FileType.WAL, ServerConstants.getVolumeReplacements());
 +        if (switchedWalog != null) {
 +          // replaces the volume used for sorting, but do not change entry in metadata table. When the tablet loads it will change the metadata table entry. If
 +          // the tablet has the same replacement config, then it will find the sorted log.
 +          log.info("Volume replaced " + walog + " -> " + switchedWalog);
 +          walog = switchedWalog;
 +        }
 +
 +        String parts[] = walog.split("/");
 +        String sortId = parts[parts.length - 1];
 +        String filename = master.getFileSystem().getFullPath(FileType.WAL, walog).toString();
 +        String dest = RecoveryPath.getRecoveryPath(master.getFileSystem(), new Path(filename)).toString();
 +        log.debug("Recovering " + filename + " to " + dest);
 +
 +        boolean sortQueued;
 +        synchronized (this) {
 +          sortQueued = sortsQueued.contains(sortId);
 +        }
 +
 +        if (sortQueued && zooCache.get(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId) == null) {
 +          synchronized (this) {
 +            sortsQueued.remove(sortId);
 +          }
 +        }
 +
 +        if (master.getFileSystem().exists(SortedLogState.getFinishedMarkerPath(dest))) {
 +          synchronized (this) {
 +            closeTasksQueued.remove(sortId);
 +            recoveryDelay.remove(sortId);
 +            sortsQueued.remove(sortId);
 +          }
 +          continue;
 +        }
 +
 +        recoveryNeeded = true;
 +        synchronized (this) {
 +          if (!closeTasksQueued.contains(sortId) && !sortsQueued.contains(sortId)) {
 +            AccumuloConfiguration aconf = master.getConfiguration().getConfiguration();
 +            LogCloser closer = aconf.instantiateClassProperty(Property.MASTER_WALOG_CLOSER_IMPLEMETATION, LogCloser.class, new HadoopLogCloser());
 +            Long delay = recoveryDelay.get(sortId);
 +            if (delay == null) {
 +              delay = master.getSystemConfiguration().getTimeInMillis(Property.MASTER_RECOVERY_DELAY);
 +            } else {
 +              delay = Math.min(2 * delay, 1000 * 60 * 5l);
 +            }
 +
 +            log.info("Starting recovery of " + filename + " (in : " + (delay / 1000) + "s), tablet " + extent + " holds a reference");
 +
 +            executor.schedule(new LogSortTask(closer, filename, dest, sortId), delay, TimeUnit.MILLISECONDS);
 +            closeTasksQueued.add(sortId);
 +            recoveryDelay.put(sortId, delay);
 +          }
 +        }
 +      }
 +    }
 +    return recoveryNeeded;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
index f981bae,0000000..a674557
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
@@@ -1,49 -1,0 +1,51 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +public class SetGoalState {
 +  
 +  /**
 +   * Utility program that will change the goal state for the master from the command line.
 +   */
 +  public static void main(String[] args) throws Exception {
 +    if (args.length != 1 || MasterGoalState.valueOf(args[0]) == null) {
 +      System.err.println("Usage: accumulo " + SetGoalState.class.getName() + " [NORMAL|SAFE_MODE|CLEAN_STOP]");
 +      System.exit(-1);
 +    }
 +    SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Accumulo.waitForZookeeperAndHdfs(fs);
-     ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(Constants.UTF8),
++    ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(UTF_8),
 +        NodeExistsPolicy.OVERWRITE);
 +  }
 +  
 +}


[28/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index 665b132,0000000..4975428
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -1,356 -1,0 +1,357 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.OutputStream;
 +import java.io.OutputStreamWriter;
 +import java.io.PrintWriter;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.UUID;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +import javax.servlet.http.HttpSession;
 +
 +import jline.console.ConsoleReader;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.shell.Shell;
 +
 +public class ShellServlet extends BasicServlet {
 +  private static final long serialVersionUID = 1L;
 +  private Map<String,ShellExecutionThread> userShells = new HashMap<String,ShellExecutionThread>();
 +  private ExecutorService service = Executors.newCachedThreadPool();
 +
 +  public static final String CSRF_KEY = "csrf_token";
 +
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Shell";
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws IOException {
 +    HttpSession session = req.getSession(true);
 +    final String CSRF_TOKEN;
 +    if (null == session.getAttribute(CSRF_KEY)) {
 +      // No token, make one
 +      CSRF_TOKEN = UUID.randomUUID().toString();
 +      session.setAttribute(CSRF_KEY, CSRF_TOKEN);
 +    } else {
 +      // Pull the token out of the session
 +      CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
 +      if (null == CSRF_TOKEN) {
 +        throw new RuntimeException("No valid CSRF token exists in session");
 +      }
 +    }
 +
 +    String user = (String) session.getAttribute("user");
 +    if (user == null) {
 +      // user attribute is null, check to see if username and password are passed as parameters
 +      user = req.getParameter("user");
 +      String pass = req.getParameter("pass");
 +      String mock = req.getParameter("mock");
 +      if (user == null || pass == null) {
 +        // username or password are null, re-authenticate
 +        sb.append(authenticationForm(req.getRequestURI(), CSRF_TOKEN));
 +        return;
 +      }
 +      try {
 +        // get a new shell for this user
 +        ShellExecutionThread shellThread = new ShellExecutionThread(user, pass, mock);
 +        service.submit(shellThread);
 +        userShells.put(session.getId(), shellThread);
 +      } catch (IOException e) {
 +        // error validating user, reauthenticate
 +        sb.append("<div id='loginError'>Invalid user/password</div>" + authenticationForm(req.getRequestURI(), CSRF_TOKEN));
 +        return;
 +      }
 +      session.setAttribute("user", user);
 +    }
 +    if (!userShells.containsKey(session.getId())) {
 +      // no existing shell for this user, re-authenticate
 +      sb.append(authenticationForm(req.getRequestURI(), UUID.randomUUID().toString()));
 +      return;
 +    }
 +
 +    ShellExecutionThread shellThread = userShells.get(session.getId());
 +    shellThread.getOutput();
 +    shellThread.printInfo();
 +    sb.append("<div id='shell'>\n");
 +    sb.append("<pre id='shellResponse'>").append(shellThread.getOutput()).append("</pre>\n");
 +    sb.append("<form><span id='shellPrompt'>").append(shellThread.getPrompt());
 +    sb.append("</span><input type='text' name='cmd' id='cmd' onkeydown='return handleKeyDown(event.keyCode);'>\n");
 +    sb.append("</form>\n</div>\n");
 +    sb.append("<script type='text/javascript'>\n");
 +    sb.append("var url = '").append(req.getRequestURL().toString()).append("';\n");
 +    sb.append("var xmlhttp = new XMLHttpRequest();\n");
 +    sb.append("var hsize = 1000;\n");
 +    sb.append("var hindex = 0;\n");
 +    sb.append("var history = new Array();\n");
 +    sb.append("\n");
 +    sb.append("function handleKeyDown(keyCode) {\n");
 +    sb.append("  if (keyCode==13) {\n");
 +    sb.append("    submitCmd(document.getElementById('cmd').value);\n");
 +    sb.append("    hindex = history.length;\n");
 +    sb.append("    return false;\n");
 +    sb.append("  } else if (keyCode==38) {\n");
 +    sb.append("    hindex = hindex==0 ? history.length : hindex - 1;\n");
 +    sb.append("    if (hindex == history.length)\n");
 +    sb.append("      document.getElementById('cmd').value = '';\n");
 +    sb.append("    else\n");
 +    sb.append("      document.getElementById('cmd').value = history[hindex];\n");
 +    sb.append("    return false;\n");
 +    sb.append("  } else if (keyCode==40) {\n");
 +    sb.append("    hindex = hindex==history.length ? history.length : hindex + 1;\n");
 +    sb.append("    if (hindex == history.length)\n");
 +    sb.append("      document.getElementById('cmd').value = '';\n");
 +    sb.append("    else\n");
 +    sb.append("      document.getElementById('cmd').value = history[hindex];\n");
 +    sb.append("    return false;\n");
 +    sb.append("  }\n");
 +    sb.append("  return true;\n");
 +    sb.append("}\n");
 +    sb.append("\n");
 +    sb.append("function submitCmd(cmd) {\n");
 +    sb.append("  if (cmd=='history') {\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += document.getElementById('shellPrompt').innerHTML+cmd+'\\n';\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += history.join('\\n');\n");
 +    sb.append("    return\n");
 +    sb.append("  }\n");
 +    sb.append("  xmlhttp.open('POST',url+'?cmd='+cmd+'&'+'").append(CSRF_KEY).append("=").append(CSRF_TOKEN).append("',false);\n");
 +    sb.append("  xmlhttp.send();\n");
 +    sb.append("  var text = xmlhttp.responseText;\n");
 +    sb.append("  var index = text.lastIndexOf('\\n');\n");
 +    sb.append("  if (index >= 0) {\n");
 +    sb.append("    if (index > 0 && document.getElementById('cmd').type == 'text') {\n");
 +    sb.append("      if (history.length == hsize)\n");
 +    sb.append("        history.shift()\n");
 +    sb.append("      history.push(cmd)\n");
 +    sb.append("    }\n");
 +    sb.append("    if (text.charAt(text.length-1)=='*') {\n");
 +    sb.append("      document.getElementById('cmd').type = 'password';\n");
 +    sb.append("      text = text.substring(0,xmlhttp.responseText.length-2);\n");
 +    sb.append("    } else {\n");
 +    sb.append("      document.getElementById('cmd').type = 'text';\n");
 +    sb.append("    }\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += text.substring(0,index+1);\n");
 +    sb.append("    document.getElementById('shellPrompt').innerHTML = text.substring(index+1);\n");
 +    sb.append("    document.getElementById('cmd').value = '';\n");
 +    sb.append("    document.getElementById('shell').scrollTop = document.getElementById('cmd').offsetTop;\n");
 +    sb.append("  } else {\n");
 +    sb.append("    window.location = url;\n");
 +    sb.append("  }\n");
 +    sb.append("}\n");
 +    sb.append("</script>\n");
 +    sb.append("<script type='text/javascript'>window.onload = function() { document.getElementById('cmd').select(); }</script>\n");
 +  }
 +
 +  @Override
 +  protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    final HttpSession session = req.getSession(true);
 +    String user = (String) session.getAttribute("user");
 +    if (user == null || !userShells.containsKey(session.getId())) {
 +      // no existing shell for user, re-authenticate
 +      doGet(req, resp);
 +      return;
 +    }
 +    final String CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
 +    if (null == CSRF_TOKEN) {
 +      // no csrf token, need to re-auth
 +      doGet(req, resp);
 +    }
 +    ShellExecutionThread shellThread = userShells.get(session.getId());
 +    String cmd = req.getParameter("cmd");
 +    if (cmd == null) {
 +      // the command is null, just print prompt
 +      resp.getWriter().append(shellThread.getPrompt());
 +      resp.getWriter().flush();
 +      return;
 +    }
 +    shellThread.addInputString(cmd);
 +    shellThread.waitUntilReady();
 +    if (shellThread.isDone()) {
 +      // the command was exit, invalidate session
 +      userShells.remove(session.getId());
 +      session.invalidate();
 +      return;
 +    }
 +    // get the shell's output
 +    StringBuilder sb = new StringBuilder();
 +    sb.append(shellThread.getOutput().replace("<", "&lt;").replace(">", "&gt;"));
 +    if (sb.length() == 0 || !(sb.charAt(sb.length() - 1) == '\n'))
 +      sb.append("\n");
 +    // check if shell is waiting for input
 +    if (!shellThread.isWaitingForInput())
 +      sb.append(shellThread.getPrompt());
 +    // check if shell is waiting for password input
 +    if (shellThread.isMasking())
 +      sb.append("*");
 +    resp.getWriter().append(sb.toString());
 +    resp.getWriter().flush();
 +  }
 +
 +  private String authenticationForm(String requestURI, String csrfToken) {
 +    return "<div id='login'><form method=POST action='" + requestURI + "'>"
 +        + "<table><tr><td>Mock:&nbsp</td><td><input type='checkbox' name='mock' value='mock'></td></tr>"
 +        + "<tr><td>Username:&nbsp;</td><td><input type='text' name='user'></td></tr>"
 +        + "<tr><td>Password:&nbsp;</td><td><input type='password' name='pass'></td><td>"
 +        + "<input type='hidden' name='" + CSRF_KEY + "' value='" + csrfToken + "'/><input type='submit' value='Enter'></td></tr></table></form></div>";
 +  }
 +  
 +  private static class StringBuilderOutputStream extends OutputStream {
 +    StringBuilder sb = new StringBuilder();
 +    
 +    @Override
 +    public void write(int b) throws IOException {
 +      sb.append((char) (0xff & b));
 +    }
 +    
 +    public String get() {
 +      return sb.toString();
 +    }
 +    
 +    public void clear() {
 +      sb.setLength(0);
 +    }
 +  }
 +  
 +  private static class ShellExecutionThread extends InputStream implements Runnable {
 +    private Shell shell;
 +    StringBuilderOutputStream output;
 +    private String cmd;
 +    private int cmdIndex;
 +    private boolean done;
 +    private boolean readWait;
 +    
 +    private ShellExecutionThread(String username, String password, String mock) throws IOException {
 +      this.done = false;
 +      this.cmd = null;
 +      this.cmdIndex = 0;
 +      this.readWait = false;
 +      this.output = new StringBuilderOutputStream();
 +      ConsoleReader reader = new ConsoleReader(this, output);
-       this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, Constants.UTF8)));
++      this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, UTF_8)));
 +      shell.setLogErrorsToConsole();
 +      if (mock != null) {
 +        if (shell.config("--fake", "-u", username, "-p", password))
 +          throw new IOException("mock shell config error");
 +      } else if (shell.config("-u", username, "-p", password)) {
 +        throw new IOException("shell config error");
 +      }
 +    }
 +    
 +    @Override
 +    public synchronized int read() throws IOException {
 +      if (cmd == null) {
 +        readWait = true;
 +        this.notifyAll();
 +      }
 +      while (cmd == null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {}
 +      }
 +      readWait = false;
 +      int c;
 +      if (cmdIndex == cmd.length())
 +        c = '\n';
 +      else
 +        c = cmd.charAt(cmdIndex);
 +      cmdIndex++;
 +      if (cmdIndex > cmd.length()) {
 +        cmd = null;
 +        cmdIndex = 0;
 +        this.notifyAll();
 +      }
 +      return c;
 +    }
 +    
 +    @Override
 +    public synchronized void run() {
 +      Thread.currentThread().setName("shell thread");
 +      while (!shell.hasExited()) {
 +        while (cmd == null) {
 +          try {
 +            this.wait();
 +          } catch (InterruptedException e) {}
 +        }
 +        String tcmd = cmd;
 +        cmd = null;
 +        cmdIndex = 0;
 +        try {
 +          shell.execCommand(tcmd, false, true);
 +        } catch (IOException e) {}
 +        this.notifyAll();
 +      }
 +      done = true;
 +      this.notifyAll();
 +    }
 +    
 +    public synchronized void addInputString(String s) {
 +      if (done)
 +        throw new IllegalStateException("adding string to exited shell");
 +      if (cmd == null) {
 +        cmd = s;
 +      } else {
 +        throw new IllegalStateException("adding string to shell not waiting for input");
 +      }
 +      this.notifyAll();
 +    }
 +    
 +    public synchronized void waitUntilReady() {
 +      while (cmd != null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +    
 +    public synchronized String getOutput() {
 +      String s = output.get();
 +      output.clear();
 +      return s;
 +    }
 +    
 +    public String getPrompt() {
 +      return shell.getDefaultPrompt();
 +    }
 +    
 +    public void printInfo() throws IOException {
 +      shell.printInfo();
 +    }
 +    
 +    public boolean isMasking() {
 +      return shell.isMasking();
 +    }
 +    
 +    public synchronized boolean isWaitingForInput() {
 +      return readWait;
 +    }
 +    
 +    public boolean isDone() {
 +      return done;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
index 0cbf957,0000000..3cd635e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
@@@ -1,179 -1,0 +1,180 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Map.Entry;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.master.thrift.Compacting;
 +import org.apache.accumulo.core.master.thrift.DeadServer;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.master.state.TabletServerState;
 +import org.apache.accumulo.server.util.TableInfoUtil;
 +
 +public class XMLServlet extends BasicServlet {
 +  private static final long serialVersionUID = 1L;
 +  
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "XML Report";
 +  }
 +  
 +  @Override
 +  protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
-     resp.setContentType("text/xml;charset=" + Constants.UTF8.name());
-     sb.append("<?xml version=\"1.0\" encoding=\"" + Constants.UTF8.name() + "\"?>\n");
++    resp.setContentType("text/xml;charset=" + UTF_8.name());
++    sb.append("<?xml version=\"1.0\" encoding=\"" + UTF_8.name() + "\"?>\n");
 +    sb.append("<stats>\n");
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
 +    double totalIngest = 0.;
 +    double totalQuery = 0.;
 +    double disk = 0.0;
 +    long totalEntries = 0L;
 +    
 +    sb.append("\n<servers>\n");
 +    if (Monitor.getMmi() == null || Monitor.getMmi().tableMap == null) {
 +      sb.append("</servers>\n");
 +      return;
 +    }
 +    SortedMap<String,TableInfo> tableStats = new TreeMap<String,TableInfo>(Monitor.getMmi().tableMap);
 +    
 +    for (TabletServerStatus status : Monitor.getMmi().tServerInfo) {
 +      
 +      sb.append("\n<server id='").append(status.name).append("'>\n");
 +      sb.append("<hostname>").append(TServerLinkType.displayName(status.name)).append("</hostname>");
 +      sb.append("<lastContact>").append(System.currentTimeMillis() - status.lastContact).append("</lastContact>\n");
 +      sb.append("<osload>").append(status.osLoad).append("</osload>\n");
 +      
 +      TableInfo summary = TableInfoUtil.summarizeTableStats(status);
 +      sb.append("<compactions>\n");
 +      sb.append("<major>").append("<running>").append(summary.majors.running).append("</running>").append("<queued>").append(summary.majors.queued)
 +          .append("</queued>").append("</major>\n");
 +      sb.append("<minor>").append("<running>").append(summary.minors.running).append("</running>").append("<queued>").append(summary.minors.queued)
 +          .append("</queued>").append("</minor>\n");
 +      sb.append("</compactions>\n");
 +      
 +      sb.append("<tablets>").append(summary.tablets).append("</tablets>\n");
 +      
 +      sb.append("<ingest>").append(summary.ingestRate).append("</ingest>\n");
 +      sb.append("<query>").append(summary.queryRate).append("</query>\n");
 +      sb.append("<ingestMB>").append(summary.ingestByteRate / 1000000.0).append("</ingestMB>\n");
 +      sb.append("<queryMB>").append(summary.queryByteRate / 1000000.0).append("</queryMB>\n");
 +      sb.append("<scans>").append(summary.scans.running + summary.scans.queued).append("</scans>");
 +      sb.append("<scansessions>").append(Monitor.getLookupRate()).append("</scansessions>\n");
 +      sb.append("<holdtime>").append(status.holdTime).append("</holdtime>\n");
 +      totalIngest += summary.ingestRate;
 +      totalQuery += summary.queryRate;
 +      totalEntries += summary.recs;
 +      sb.append("</server>\n");
 +    }
 +    sb.append("\n</servers>\n");
 +    
 +    sb.append("\n<masterGoalState>" + Monitor.getMmi().goalState + "</masterGoalState>\n");
 +    sb.append("\n<masterState>" + Monitor.getMmi().state + "</masterState>\n");
 +    
 +    sb.append("\n<badTabletServers>\n");
 +    for (Entry<String,Byte> entry : Monitor.getMmi().badTServers.entrySet()) {
 +      sb.append(String.format("<badTabletServer id='%s' status='%s'/>\n", entry.getKey(), TabletServerState.getStateById(entry.getValue())));
 +    }
 +    sb.append("\n</badTabletServers>\n");
 +    
 +    sb.append("\n<tabletServersShuttingDown>\n");
 +    for (String server : Monitor.getMmi().serversShuttingDown) {
 +      sb.append(String.format("<server id='%s'/>\n", server));
 +    }
 +    sb.append("\n</tabletServersShuttingDown>\n");
 +    
 +    sb.append(String.format("\n<unassignedTablets>%d</unassignedTablets>\n", Monitor.getMmi().unassignedTablets));
 +    
 +    sb.append("\n<deadTabletServers>\n");
 +    for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
 +      sb.append(String.format("<deadTabletServer id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
 +    }
 +    sb.append("\n</deadTabletServers>\n");
 +    
 +    sb.append("\n<deadLoggers>\n");
 +    for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
 +      sb.append(String.format("<deadLogger id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
 +    }
 +    sb.append("\n</deadLoggers>\n");
 +    
 +    sb.append("\n<tables>\n");
 +    Instance instance = HdfsZooInstance.getInstance();
 +    for (Entry<String,TableInfo> entry : tableStats.entrySet()) {
 +      TableInfo tableInfo = entry.getValue();
 +      
 +      sb.append("\n<table>\n");
 +      String tableId = entry.getKey();
 +      String tableName = "unknown";
 +      String tableState = "unknown";
 +      try {
 +        tableName = Tables.getTableName(instance, tableId);
 +        tableState = Tables.getTableState(instance, tableId).toString();
 +      } catch (Exception ex) {
 +        log.warn(ex, ex);
 +      }
 +      sb.append("<tablename>").append(tableName).append("</tablename>\n");
 +      sb.append("<tableId>").append(tableId).append("</tableId>\n");
 +      sb.append("<tableState>").append(tableState).append("</tableState>\n");
 +      sb.append("<tablets>").append(tableInfo.tablets).append("</tablets>\n");
 +      sb.append("<onlineTablets>").append(tableInfo.onlineTablets).append("</onlineTablets>\n");
 +      sb.append("<recs>").append(tableInfo.recs).append("</recs>\n");
 +      sb.append("<recsInMemory>").append(tableInfo.recsInMemory).append("</recsInMemory>\n");
 +      sb.append("<ingest>").append(tableInfo.ingestRate).append("</ingest>\n");
 +      sb.append("<ingestByteRate>").append(tableInfo.ingestByteRate).append("</ingestByteRate>\n");
 +      sb.append("<query>").append(tableInfo.queryRate).append("</query>\n");
 +      sb.append("<queryByteRate>").append(tableInfo.queryRate).append("</queryByteRate>\n");
 +      int running = 0;
 +      int queued = 0;
 +      Compacting compacting = entry.getValue().majors;
 +      if (compacting != null) {
 +        running = compacting.running;
 +        queued = compacting.queued;
 +      }
 +      sb.append("<majorCompactions>").append("<running>").append(running).append("</running>").append("<queued>").append(queued).append("</queued>")
 +          .append("</majorCompactions>\n");
 +      sb.append("</table>\n");
 +    }
 +    sb.append("\n</tables>\n");
 +    
 +    sb.append("\n<totals>\n");
 +    sb.append("<ingestrate>").append(totalIngest).append("</ingestrate>\n");
 +    sb.append("<queryrate>").append(totalQuery).append("</queryrate>\n");
 +    sb.append("<diskrate>").append(disk).append("</diskrate>\n");
 +    sb.append("<numentries>").append(totalEntries).append("</numentries>\n");
 +    sb.append("</totals>\n");
 +  }
 +  
 +  @Override
 +  protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
 +    sb.append("\n</stats>\n");
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 90bb0d4,0000000..1c464d4
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@@ -1,105 -1,0 +1,106 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets.trace;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Date;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +
 +import javax.servlet.http.HttpServletRequest;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.trace.TraceFormatter;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.servlets.BasicServlet;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +
 +abstract class Basic extends BasicServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  public static String getStringParameter(HttpServletRequest req, String name, String defaultValue) {
 +    String result = req.getParameter(name);
 +    if (result == null) {
 +      return defaultValue;
 +    }
 +    return result;
 +  }
 +
 +  public static int getIntParameter(HttpServletRequest req, String name, int defaultMinutes) {
 +    String valueString = req.getParameter(name);
 +    if (valueString == null)
 +      return defaultMinutes;
 +    int result = 0;
 +    try {
 +      result = Integer.parseInt(valueString);
 +    } catch (NumberFormatException ex) {
 +      return defaultMinutes;
 +    }
 +    return result;
 +  }
 +
 +  public static String dateString(long millis) {
 +    return TraceFormatter.formatDate(new Date(millis));
 +  }
 +
 +  protected Scanner getScanner(StringBuilder sb) throws AccumuloException, AccumuloSecurityException {
 +    AccumuloConfiguration conf = Monitor.getSystemConfiguration();
 +    String principal = conf.get(Property.TRACE_USER);
 +    AuthenticationToken at;
 +    Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
 +    if (loginMap.isEmpty()) {
 +      Property p = Property.TRACE_PASSWORD;
-       at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++      at = new PasswordToken(conf.get(p).getBytes(UTF_8));
 +    } else {
 +      Properties props = new Properties();
 +      int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
 +      for (Entry<String,String> entry : loginMap.entrySet()) {
 +        props.put(entry.getKey().substring(prefixLength), entry.getValue());
 +      }
 +
 +      AuthenticationToken token = Property.createInstanceFromPropertyName(conf, Property.TRACE_TOKEN_TYPE, AuthenticationToken.class, new PasswordToken());
 +      token.init(props);
 +      at = token;
 +    }
 +
 +    String table = conf.get(Property.TRACE_TABLE);
 +    try {
 +      Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
 +      if (!conn.tableOperations().exists(table)) {
 +        return new NullScanner();
 +      }
 +      Scanner scanner = conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
 +      return scanner;
 +    } catch (AccumuloSecurityException ex) {
 +      sb.append("<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
 +      return null;
 +    } catch (TableNotFoundException ex) {
 +      return new NullScanner();
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
----------------------------------------------------------------------
diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index e3ae0c8,0000000..6094911
mode 100644,000000..100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -1,324 -1,0 +1,326 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tracer;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.net.InetSocketAddress;
 +import java.net.ServerSocket;
 +import java.nio.channels.ServerSocketChannel;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.atomic.AtomicReference;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.user.AgeOffFilter;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.trace.TraceFormatter;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.thrift.RemoteSpan;
 +import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
 +import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TByteArrayOutputStream;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.protocol.TCompactProtocol;
 +import org.apache.thrift.server.TServer;
 +import org.apache.thrift.server.TThreadPoolServer;
 +import org.apache.thrift.transport.TServerSocket;
 +import org.apache.thrift.transport.TServerTransport;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.Watcher.Event.EventType;
 +import org.apache.zookeeper.Watcher.Event.KeeperState;
 +
 +public class TraceServer implements Watcher {
 +
 +  final private static Logger log = Logger.getLogger(TraceServer.class);
 +  final private ServerConfiguration serverConfiguration;
 +  final private TServer server;
 +  final private AtomicReference<BatchWriter> writer;
 +  final private Connector connector;
 +  final String table;
 +
 +  private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) {
 +    m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
 +  }
 +
 +  static class ByteArrayTransport extends TTransport {
 +    TByteArrayOutputStream out = new TByteArrayOutputStream();
 +
 +    @Override
 +    public boolean isOpen() {
 +      return true;
 +    }
 +
 +    @Override
 +    public void open() throws TTransportException {}
 +
 +    @Override
 +    public void close() {}
 +
 +    @Override
 +    public int read(byte[] buf, int off, int len) {
 +      return 0;
 +    }
 +
 +    @Override
 +    public void write(byte[] buf, int off, int len) throws TTransportException {
 +      out.write(buf, off, len);
 +    }
 +
 +    public byte[] get() {
 +      return out.get();
 +    }
 +
 +    public int len() {
 +      return out.len();
 +    }
 +  }
 +
 +  class Receiver implements Iface {
 +    @Override
 +    public void span(RemoteSpan s) throws TException {
 +      String idString = Long.toHexString(s.traceId);
 +      String startString = Long.toHexString(s.start);
 +      Mutation spanMutation = new Mutation(new Text(idString));
 +      Mutation indexMutation = new Mutation(new Text("idx:" + s.svc + ":" + startString));
 +      long diff = s.stop - s.start;
-       indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(Constants.UTF8)));
++      indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(UTF_8)));
 +      ByteArrayTransport transport = new ByteArrayTransport();
 +      TCompactProtocol protocol = new TCompactProtocol(transport);
 +      s.write(protocol);
 +      String parentString = Long.toHexString(s.parentId);
 +      if (s.parentId == Span.ROOT_SPAN_ID)
 +        parentString = "";
 +      put(spanMutation, "span", parentString + ":" + Long.toHexString(s.spanId), transport.get(), transport.len());
 +      // Map the root span to time so we can look up traces by time
 +      Mutation timeMutation = null;
 +      if (s.parentId == Span.ROOT_SPAN_ID) {
 +        timeMutation = new Mutation(new Text("start:" + startString));
 +        put(timeMutation, "id", idString, transport.get(), transport.len());
 +      }
 +      try {
 +        final BatchWriter writer = TraceServer.this.writer.get();
 +        /*
 +         * Check for null, because we expect spans to come in much faster than flush calls. In the case of failure, we'd rather avoid logging tons of NPEs.
 +         */
 +        if (null == writer) {
 +          log.warn("writer is not ready; discarding span.");
 +          return;
 +        }
 +        writer.addMutation(spanMutation);
 +        writer.addMutation(indexMutation);
 +        if (timeMutation != null)
 +          writer.addMutation(timeMutation);
 +      } catch (MutationsRejectedException exception) {
 +        log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception);
 +        if (log.isDebugEnabled()) {
 +          log.debug("discarded span due to rejection of mutation: " + spanMutation, exception);
 +        }
 +        /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */
 +      } catch (RuntimeException exception) {
 +        log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception);
 +        log.debug("unable to write mutation to table due to exception.", exception);
 +      }
 +    }
 +
 +  }
 +
 +  public TraceServer(ServerConfiguration serverConfiguration, String hostname) throws Exception {
 +    this.serverConfiguration = serverConfiguration;
 +    AccumuloConfiguration conf = serverConfiguration.getConfiguration();
 +    table = conf.get(Property.TRACE_TABLE);
 +    Connector connector = null;
 +    while (true) {
 +      try {
 +        String principal = conf.get(Property.TRACE_USER);
 +        AuthenticationToken at;
 +        Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
 +        if (loginMap.isEmpty()) {
 +          Property p = Property.TRACE_PASSWORD;
-           at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++          at = new PasswordToken(conf.get(p).getBytes(UTF_8));
 +        } else {
 +          Properties props = new Properties();
 +          AuthenticationToken token = AccumuloVFSClassLoader.getClassLoader().loadClass(conf.get(Property.TRACE_TOKEN_TYPE)).asSubclass(AuthenticationToken.class)
 +              .newInstance();
 +
 +          int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
 +          for (Entry<String,String> entry : loginMap.entrySet()) {
 +            props.put(entry.getKey().substring(prefixLength), entry.getValue());
 +          }
 +
 +          token.init(props);
 +
 +          at = token;
 +        }
 +
 +        connector = serverConfiguration.getInstance().getConnector(principal, at);
 +        if (!connector.tableOperations().exists(table)) {
 +          connector.tableOperations().create(table);
 +          IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName());
 +          AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
 +          connector.tableOperations().attachIterator(table, setting);
 +        }
 +        connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
 +        break;
 +      } catch (Exception ex) {
 +        log.info("Waiting to checking/create the trace table.", ex);
 +        UtilWaitThread.sleep(1000);
 +      }
 +    }
 +    this.connector = connector;
 +    // make sure we refer to the final variable from now on.
 +    connector = null;
 +
 +    int port = conf.getPort(Property.TRACE_PORT);
 +    final ServerSocket sock = ServerSocketChannel.open().socket();
 +    sock.setReuseAddress(true);
 +    sock.bind(new InetSocketAddress(hostname, port));
 +    final TServerTransport transport = new TServerSocket(sock);
 +    TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
 +    options.processor(new Processor<Iface>(new Receiver()));
 +    server = new TThreadPoolServer(options);
 +    registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort());
 +    writer = new AtomicReference<BatchWriter>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS)));
 +  }
 +
 +  public void run() throws Exception {
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        flush();
 +      }
 +    }, 1000, 1000);
 +    server.serve();
 +  }
 +
 +  private void flush() {
 +    try {
 +      final BatchWriter writer = this.writer.get();
 +      if (null != writer) {
 +        writer.flush();
 +      }
 +    } catch (MutationsRejectedException exception) {
 +      log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
 +      log.debug("flushing traces failed due to exception", exception);
 +      resetWriter();
 +      /* XXX e.g. if the writer was closed between when we grabbed it and when we called flush. */
 +    } catch (RuntimeException exception) {
 +      log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
 +      log.debug("flushing traces failed due to exception", exception);
 +      resetWriter();
 +    }
 +  }
 +
 +  private void resetWriter() {
 +    BatchWriter writer = null;
 +    try {
 +      writer = connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS));
 +    } catch (Exception ex) {
 +      log.warn("Unable to create a batch writer, will retry. Set log level to DEBUG to see stacktrace. cause: " + ex);
 +      log.debug("batch writer creation failed with exception.", ex);
 +    } finally {
 +      /* Trade in the new writer (even if null) for the one we need to close. */
 +      writer = this.writer.getAndSet(writer);
 +      try {
 +        if (null != writer) {
 +          writer.close();
 +        }
 +      } catch (Exception ex) {
 +        log.warn("Problem closing batch writer. Set log level to DEBUG to see stacktrace. cause: " + ex);
 +        log.debug("batch writer close failed with exception", ex);
 +      }
 +    }
 +  }
 +
 +  private void registerInZooKeeper(String name) throws Exception {
 +    String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-     String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(Constants.UTF8));
++    String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(UTF_8));
 +    zoo.exists(path, this);
 +  }
 +
 +  public static void main(String[] args) throws Exception {
 +    SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +    ServerOpts opts = new ServerOpts();
 +    final String app = "tracer";
 +    opts.parseArgs(app, args);
 +    Accumulo.setupLogging(app);
 +    Instance instance = HdfsZooInstance.getInstance();
 +    ServerConfiguration conf = new ServerConfiguration(instance);
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Accumulo.init(fs, conf, app);
 +    String hostname = opts.getAddress();
 +    TraceServer server = new TraceServer(conf, hostname);
 +    Accumulo.enableTracing(hostname, app);
 +    server.run();
 +    log.info("tracer stopping");
 +  }
 +
 +  @Override
 +  public void process(WatchedEvent event) {
 +    log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
 +    if (event.getState() == KeeperState.Expired) {
 +      log.warn("Trace server lost zookeeper registration at " + event.getPath());
 +      server.stop();
 +    } else if (event.getType() == EventType.NodeDeleted) {
 +      log.warn("Trace server zookeeper entry lost " + event.getPath());
 +      server.stop();
 +    }
 +    if (event.getPath() != null) {
 +      try {
 +        if (ZooReaderWriter.getInstance().exists(event.getPath(), this))
 +          return;
 +      } catch (Exception ex) {
 +        log.error(ex, ex);
 +      }
 +      log.warn("Trace server unable to reset watch on zookeeper registration");
 +      server.stop();
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index 39bf81e,0000000..4d9a6c2
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@@ -1,75 -1,0 +1,76 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.trace.TraceFileSystem;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.FileUtil;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * Copy failed bulk imports.
 + */
 +public class BulkFailedCopyProcessor implements Processor {
 +
 +  private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
 +
 +  @Override
 +  public Processor newProcessor() {
 +    return new BulkFailedCopyProcessor();
 +  }
 +
 +  @Override
 +  public void process(String workID, byte[] data) {
 +
-     String paths[] = new String(data, Constants.UTF8).split(",");
++    String paths[] = new String(data, UTF_8).split(",");
 +
 +    Path orig = new Path(paths[0]);
 +    Path dest = new Path(paths[1]);
 +    Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
 +
 +    try {
 +      VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
 +      FileSystem origFs = TraceFileSystem.wrap(vm.getVolumeByPath(orig).getFileSystem());
 +      FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
 +      
 +      FileUtil.copy(origFs, orig, destFs, tmp, false, true, CachedConfiguration.getInstance());
 +      destFs.rename(tmp, dest);
 +      log.debug("copied " + orig + " to " + dest);
 +    } catch (IOException ex) {
 +      try {
 +        VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
 +        FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
 +        destFs.create(dest).close();
 +        log.warn(" marked " + dest + " failed", ex);
 +      } catch (IOException e) {
 +        log.error("Unable to create failure flag file " + dest, e);
 +      }
 +    }
 +
 +  }
 +
 +}


[23/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a338d5f,0000000..d5c940d
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@@ -1,553 -1,0 +1,553 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.BufferedOutputStream;
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.net.URI;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.admin.DiskUsage;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.init.Initialize;
 +import org.apache.accumulo.server.util.Admin;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.fs.RawLocalFileSystem;
 +import org.apache.hadoop.io.Text;
 +import org.apache.zookeeper.ZooKeeper;
 +import org.junit.Assert;
 +import org.junit.Test;
 +
 +public class VolumeIT extends ConfigurableMacIT {
 +
 +  private static final Text EMPTY = new Text();
 +  private static final Value EMPTY_VALUE = new Value(new byte[] {});
 +  private File volDirBase;
 +  private Path v1, v2;
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 5 * 60;
 +  }
 +
 +  @SuppressWarnings("deprecation")
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    File baseDir = cfg.getDir();
 +    volDirBase = new File(baseDir, "volumes");
 +    File v1f = new File(volDirBase, "v1");
 +    File v2f = new File(volDirBase, "v2");
 +    v1f.mkdir();
 +    v2f.mkdir();
 +    v1 = new Path("file://" + v1f.getAbsolutePath());
 +    v2 = new Path("file://" + v2f.getAbsolutePath());
 +
 +    // Run MAC on two locations in the local file system
 +    URI v1Uri = v1.toUri();
 +    cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
 +    cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
 +    cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
 +
 +    // use raw local file system so walogs sync and flush will work
 +    hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
 +
 +    super.configure(cfg, hadoopCoreSite);
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    // create a table
 +    Connector connector = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    connector.tableOperations().create(tableName);
 +    SortedSet<Text> partitions = new TreeSet<Text>();
 +    // with some splits
 +    for (String s : "d,m,t".split(","))
 +      partitions.add(new Text(s));
 +    connector.tableOperations().addSplits(tableName, partitions);
 +    // scribble over the splits
 +    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
 +    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
 +    for (String s : rows) {
 +      Mutation m = new Mutation(new Text(s));
 +      m.put(EMPTY, EMPTY, EMPTY_VALUE);
 +      bw.addMutation(m);
 +    }
 +    bw.close();
 +    // write the data to disk, read it back
 +    connector.tableOperations().flush(tableName, null, null, true);
 +    Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
 +    int i = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      assertEquals(rows[i++], entry.getKey().getRow().toString());
 +    }
 +    // verify the new files are written to the different volumes
 +    scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    scanner.setRange(new Range("1", "1<"));
 +    scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    int fileCount = 0;
 +
 +    for (Entry<Key,Value> entry : scanner) {
 +      boolean inV1 = entry.getKey().getColumnQualifier().toString().contains(v1.toString());
 +      boolean inV2 = entry.getKey().getColumnQualifier().toString().contains(v2.toString());
 +      assertTrue(inV1 || inV2);
 +      fileCount++;
 +    }
 +    assertEquals(4, fileCount);
 +    List<DiskUsage> diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
 +    assertEquals(1, diskUsage.size());
 +    long usage = diskUsage.get(0).getUsage().longValue();
 +    System.out.println("usage " + usage);
 +    assertTrue(usage > 700 && usage < 800);
 +  }
 +
 +  private void verifyData(List<String> expected, Scanner createScanner) {
 +
 +    List<String> actual = new ArrayList<String>();
 +
 +    for (Entry<Key,Value> entry : createScanner) {
 +      Key k = entry.getKey();
 +      actual.add(k.getRow() + ":" + k.getColumnFamily() + ":" + k.getColumnQualifier() + ":" + entry.getValue());
 +    }
 +
 +    Collections.sort(expected);
 +    Collections.sort(actual);
 +
 +    Assert.assertEquals(expected, actual);
 +  }
 +
 +  @Test
 +  public void testRelativePaths() throws Exception {
 +
 +    List<String> expected = new ArrayList<String>();
 +
 +    Connector connector = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    connector.tableOperations().create(tableName, false);
 +
 +    String tableId = connector.tableOperations().tableIdMap().get(tableName);
 +
 +    SortedSet<Text> partitions = new TreeSet<Text>();
 +    // with some splits
 +    for (String s : "c,g,k,p,s,v".split(","))
 +      partitions.add(new Text(s));
 +
 +    connector.tableOperations().addSplits(tableName, partitions);
 +
 +    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    // create two files in each tablet
 +
 +    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
 +    for (String s : rows) {
 +      Mutation m = new Mutation(s);
 +      m.put("cf1", "cq1", "1");
 +      bw.addMutation(m);
 +      expected.add(s + ":cf1:cq1:1");
 +    }
 +
 +    bw.flush();
 +    connector.tableOperations().flush(tableName, null, null, true);
 +
 +    for (String s : rows) {
 +      Mutation m = new Mutation(s);
 +      m.put("cf1", "cq1", "2");
 +      bw.addMutation(m);
 +      expected.add(s + ":cf1:cq1:2");
 +    }
 +
 +    bw.close();
 +    connector.tableOperations().flush(tableName, null, null, true);
 +
 +    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
 +
 +    connector.tableOperations().offline(tableName, true);
 +
 +    connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
 +
 +    Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
 +    metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +
 +    BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +
 +    for (Entry<Key,Value> entry : metaScanner) {
 +      String cq = entry.getKey().getColumnQualifier().toString();
 +      if (cq.startsWith(v1.toString())) {
 +        Path path = new Path(cq);
 +        String relPath = "/" + path.getParent().getName() + "/" + path.getName();
 +        Mutation fileMut = new Mutation(entry.getKey().getRow());
 +        fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
 +        fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
 +        mbw.addMutation(fileMut);
 +      }
 +    }
 +
 +    mbw.close();
 +
 +    connector.tableOperations().online(tableName, true);
 +
 +    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
 +
 +    connector.tableOperations().compact(tableName, null, null, true, true);
 +
 +    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
 +
 +    for (Entry<Key,Value> entry : metaScanner) {
 +      String cq = entry.getKey().getColumnQualifier().toString();
 +      Path path = new Path(cq);
 +      Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
 +    }
 +
 +  }
 +
 +  @Test
 +  public void testAddVolumes() throws Exception {
 +
 +    String[] tableNames = getUniqueNames(2);
 +
 +    // grab this before shutting down cluster
 +    String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +    cluster.stop();
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    File v3f = new File(volDirBase, "v3");
 +    v3f.mkdir();
 +    Path v3 = new Path("file://" + v3f.getAbsolutePath());
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v1.toString() + "," + v2.toString() + "," + v3.toString());
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // initialize volume
 +    Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
 +
 +    // check that all volumes are initialized
 +    for (Path volumePath : Arrays.asList(v1, v2, v3)) {
 +      FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
 +      Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
 +      FileStatus[] iids = fs.listStatus(vp);
 +      Assert.assertEquals(1, iids.length);
 +      Assert.assertEquals(uuid, iids[0].getPath().getName());
 +    }
 +
 +    // start cluster and verify that new volume is used
 +    cluster.start();
 +
 +    verifyVolumesUsed(tableNames[1], false, v1, v2, v3);
 +  }
 +
 +  @Test
 +  public void testNonConfiguredVolumes() throws Exception {
 +
 +    String[] tableNames = getUniqueNames(2);
 +
 +    // grab this before shutting down cluster
 +    String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +    cluster.stop();
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    File v3f = new File(volDirBase, "v3");
 +    v3f.mkdir();
 +    Path v3 = new Path("file://" + v3f.getAbsolutePath());
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // initialize volume
 +    Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
 +
 +    // check that all volumes are initialized
 +    for (Path volumePath : Arrays.asList(v1, v2, v3)) {
 +      FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
 +      Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
 +      FileStatus[] iids = fs.listStatus(vp);
 +      Assert.assertEquals(1, iids.length);
 +      Assert.assertEquals(uuid, iids[0].getPath().getName());
 +    }
 +
 +    // start cluster and verify that new volume is used
 +    cluster.start();
 +
 +    // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
 +    List<String> expected = new ArrayList<String>();
 +    for (int i = 0; i < 100; i++) {
 +      String row = String.format("%06d", i * 100 + 3);
 +      expected.add(row + ":cf1:cq1:1");
 +    }
 +
 +    verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
 +
 +    // v1 should not have any data for tableNames[1]
 +    verifyVolumesUsed(tableNames[1], false, v2, v3);
 +  }
 +
 +  private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
 +      MutationsRejectedException {
 +    TreeSet<Text> splits = new TreeSet<Text>();
 +    for (int i = 1; i < 100; i++) {
 +      splits.add(new Text(String.format("%06d", i * 100)));
 +    }
 +
 +    conn.tableOperations().create(tableName);
 +    conn.tableOperations().addSplits(tableName, splits);
 +
 +    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    for (int i = 0; i < 100; i++) {
 +      String row = String.format("%06d", i * 100 + 3);
 +      Mutation m = new Mutation(row);
 +      m.put("cf1", "cq1", "1");
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +  }
 +
 +  private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
 +      TableExistsException, TableNotFoundException, MutationsRejectedException {
 +
 +    Connector conn = getConnector();
 +
 +    List<String> expected = new ArrayList<String>();
 +    for (int i = 0; i < 100; i++) {
 +      String row = String.format("%06d", i * 100 + 3);
 +      expected.add(row + ":cf1:cq1:1");
 +    }
 +
 +    if (!conn.tableOperations().exists(tableName)) {
 +      Assert.assertFalse(shouldExist);
 +
 +      writeData(tableName, conn);
 +
 +      verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
 +
 +      conn.tableOperations().flush(tableName, null, null, true);
 +    }
 +
 +    verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
 +
 +    String tableId = conn.tableOperations().tableIdMap().get(tableName);
 +    Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
 +    metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
 +    metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +
 +    int counts[] = new int[paths.length];
 +
 +    outer: for (Entry<Key,Value> entry : metaScanner) {
 +      String cf = entry.getKey().getColumnFamily().toString();
 +      String cq = entry.getKey().getColumnQualifier().toString();
 +
 +      String path;
 +      if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
 +        path = cq;
 +      else
 +        path = entry.getValue().toString();
 +
 +      for (int i = 0; i < paths.length; i++) {
 +        if (path.startsWith(paths[i].toString())) {
 +          counts[i]++;
 +          continue outer;
 +        }
 +      }
 +
 +      Assert.fail("Unexpected volume " + path);
 +    }
 +
 +    // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
 +    // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
 +
 +    int sum = 0;
 +    for (int count : counts) {
 +      Assert.assertTrue(count > 0);
 +      sum += count;
 +    }
 +
 +    Assert.assertEquals(200, sum);
 +  }
 +
 +  @Test
 +  public void testRemoveVolumes() throws Exception {
 +    String[] tableNames = getUniqueNames(2);
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +    cluster.stop();
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString());
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // start cluster and verify that volume was decommisioned
 +    cluster.start();
 +
 +    Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
 +    conn.tableOperations().compact(tableNames[0], null, null, true, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v2);
 +
 +    // check that root tablet is not on volume 1
 +    String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
 +    ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
-     String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++    String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
 +    Assert.assertTrue(rootTabletDir.startsWith(v2.toString()));
 +    zookeeper.close();
 +
 +    conn.tableOperations().clone(tableNames[0], tableNames[1], true, new HashMap<String,String>(), new HashSet<String>());
 +
 +    conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
 +    conn.tableOperations().flush(RootTable.NAME, null, null, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v2);
 +    verifyVolumesUsed(tableNames[1], true, v2);
 +
 +  }
 +
 +  private void testReplaceVolume(boolean cleanShutdown) throws Exception {
 +    String[] tableNames = getUniqueNames(3);
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    // write to 2nd table, but do not flush data to disk before shutdown
 +    writeData(tableNames[1], cluster.getConnector("root", ROOT_PASSWORD));
 +
 +    if (cleanShutdown)
 +      Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +
 +    cluster.stop();
 +
 +    File v1f = new File(v1.toUri());
 +    File v8f = new File(new File(v1.getParent().toUri()), "v8");
 +    Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
 +    Path v8 = new Path(v8f.toURI());
 +
 +    File v2f = new File(v2.toUri());
 +    File v9f = new File(new File(v2.getParent().toUri()), "v9");
 +    Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
 +    Path v9 = new Path(v9f.toURI());
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
 +    conf.set(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(), v1 + " " + v8 + "," + v2 + " " + v9);
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // start cluster and verify that volumes were replaced
 +    cluster.start();
 +
 +    verifyVolumesUsed(tableNames[0], true, v8, v9);
 +    verifyVolumesUsed(tableNames[1], true, v8, v9);
 +
 +    // verify writes to new dir
 +    getConnector().tableOperations().compact(tableNames[0], null, null, true, true);
 +    getConnector().tableOperations().compact(tableNames[1], null, null, true, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v8, v9);
 +    verifyVolumesUsed(tableNames[1], true, v8, v9);
 +
 +    // check that root tablet is not on volume 1 or 2
 +    String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
 +    ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
-     String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++    String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
 +    Assert.assertTrue(rootTabletDir.startsWith(v8.toString()) || rootTabletDir.startsWith(v9.toString()));
 +    zookeeper.close();
 +
 +    getConnector().tableOperations().clone(tableNames[1], tableNames[2], true, new HashMap<String,String>(), new HashSet<String>());
 +
 +    getConnector().tableOperations().flush(MetadataTable.NAME, null, null, true);
 +    getConnector().tableOperations().flush(RootTable.NAME, null, null, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v8, v9);
 +    verifyVolumesUsed(tableNames[1], true, v8, v9);
 +    verifyVolumesUsed(tableNames[2], true, v8, v9);
 +  }
 +
 +  @Test
 +  public void testCleanReplaceVolumes() throws Exception {
 +    testReplaceVolume(true);
 +  }
 +
 +  @Test
 +  public void testDirtyReplaceVolumes() throws Exception {
 +    testReplaceVolume(false);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
index f657034,0000000..05de342
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
@@@ -1,140 -1,0 +1,141 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collection;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class AddSplitIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void addSplitTest() throws Exception {
 +
 +    String tableName = getUniqueNames(1)[0];
 +    Connector c = getConnector();
 +    c.tableOperations().create(tableName);
 +
 +    insertData(tableName, 1l);
 +
 +    TreeSet<Text> splits = new TreeSet<Text>();
 +    splits.add(new Text(String.format("%09d", 333)));
 +    splits.add(new Text(String.format("%09d", 666)));
 +
 +    c.tableOperations().addSplits(tableName, splits);
 +
 +    UtilWaitThread.sleep(100);
 +
 +    Collection<Text> actualSplits = c.tableOperations().listSplits(tableName);
 +
 +    if (!splits.equals(new TreeSet<Text>(actualSplits))) {
 +      throw new Exception(splits + " != " + actualSplits);
 +    }
 +
 +    verifyData(tableName, 1l);
 +    insertData(tableName, 2l);
 +
 +    // did not clear splits on purpose, it should ignore existing split points
 +    // and still create the three additional split points
 +
 +    splits.add(new Text(String.format("%09d", 200)));
 +    splits.add(new Text(String.format("%09d", 500)));
 +    splits.add(new Text(String.format("%09d", 800)));
 +
 +    c.tableOperations().addSplits(tableName, splits);
 +
 +    UtilWaitThread.sleep(100);
 +
 +    actualSplits = c.tableOperations().listSplits(tableName);
 +
 +    if (!splits.equals(new TreeSet<Text>(actualSplits))) {
 +      throw new Exception(splits + " != " + actualSplits);
 +    }
 +
 +    verifyData(tableName, 2l);
 +  }
 +
 +  private void verifyData(String tableName, long ts) throws Exception {
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +    for (int i = 0; i < 10000; i++) {
 +      if (!iter.hasNext()) {
 +        throw new Exception("row " + i + " not found");
 +      }
 +
 +      Entry<Key,Value> entry = iter.next();
 +
 +      String row = String.format("%09d", i);
 +
 +      if (!entry.getKey().getRow().equals(new Text(row))) {
 +        throw new Exception("unexpected row " + entry.getKey() + " " + i);
 +      }
 +
 +      if (entry.getKey().getTimestamp() != ts) {
 +        throw new Exception("unexpected ts " + entry.getKey() + " " + ts);
 +      }
 +
 +      if (Integer.parseInt(entry.getValue().toString()) != i) {
 +        throw new Exception("unexpected value " + entry + " " + i);
 +      }
 +    }
 +
 +    if (iter.hasNext()) {
 +      throw new Exception("found more than expected " + iter.next());
 +    }
 +
 +  }
 +
 +  private void insertData(String tableName, long ts) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, null);
 +
 +    for (int i = 0; i < 10000; i++) {
 +      String row = String.format("%09d", i);
 +
 +      Mutation m = new Mutation(new Text(row));
-       m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
index 0139a10,0000000..9c4492e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
@@@ -1,110 -1,0 +1,110 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.EnumSet;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BadIteratorMincIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    Connector c = getConnector();
 +
 +    String tableName = getUniqueNames(1)[0];
 +    c.tableOperations().create(tableName);
 +    IteratorSetting is = new IteratorSetting(30, BadIterator.class);
 +    c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    Mutation m = new Mutation(new Text("r1"));
-     m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++    m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
 +
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    c.tableOperations().flush(tableName, null, null, false);
 +    UtilWaitThread.sleep(1000);
 +
 +    // minc should fail, so there should be no files
 +    FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 0, 0);
 +
 +    // try to scan table
 +    Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
 +    int count = FunctionalTestUtils.count(scanner);
 +    assertEquals("Did not see expected # entries " + count, 1, count);
 +
 +    // remove the bad iterator
 +    c.tableOperations().removeIterator(tableName, BadIterator.class.getSimpleName(), EnumSet.of(IteratorScope.minc));
 +
 +    UtilWaitThread.sleep(5000);
 +
 +    // minc should complete
 +    FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 1, 1);
 +
 +    count = 0;
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +
 +    if (count != 1)
 +      throw new Exception("Did not see expected # entries " + count);
 +
 +    // now try putting bad iterator back and deleting the table
 +    c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
 +    bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    m = new Mutation(new Text("r2"));
-     m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++    m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    // make sure property is given time to propagate
 +    UtilWaitThread.sleep(500);
 +
 +    c.tableOperations().flush(tableName, null, null, false);
 +
 +    // make sure the flush has time to start
 +    UtilWaitThread.sleep(1000);
 +
 +    // this should not hang
 +    c.tableOperations().delete(tableName);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index c67407d,0000000..688a326
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@@ -1,125 -1,0 +1,126 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BatchScanSplitIT extends ConfigurableMacIT {
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "0"));
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    Connector c = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    c.tableOperations().create(tableName);
 +
 +    int numRows = 1 << 18;
 +
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    for (int i = 0; i < numRows; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%09x", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +
 +    getConnector().tableOperations().flush(tableName, null, null, true);
 +
 +    getConnector().tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "4K");
 +
 +    Collection<Text> splits = getConnector().tableOperations().listSplits(tableName);
 +    while (splits.size() < 2) {
 +      UtilWaitThread.sleep(1);
 +      splits = getConnector().tableOperations().listSplits(tableName);
 +    }
 +
 +    System.out.println("splits : " + splits);
 +
 +    Random random = new Random(19011230);
 +    HashMap<Text,Value> expected = new HashMap<Text,Value>();
 +    ArrayList<Range> ranges = new ArrayList<Range>();
 +    for (int i = 0; i < 100; i++) {
 +      int r = random.nextInt(numRows);
 +      Text row = new Text(String.format("%09x", r));
-       expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(Constants.UTF8)));
++      expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(UTF_8)));
 +      ranges.add(new Range(row));
 +    }
 +
 +    // logger.setLevel(Level.TRACE);
 +
 +    HashMap<Text,Value> found = new HashMap<Text,Value>();
 +
 +    for (int i = 0; i < 20; i++) {
 +      BatchScanner bs = getConnector().createBatchScanner(tableName, Authorizations.EMPTY, 4);
 +
 +      found.clear();
 +
 +      long t1 = System.currentTimeMillis();
 +
 +      bs.setRanges(ranges);
 +
 +      for (Entry<Key,Value> entry : bs) {
 +        found.put(entry.getKey().getRow(), entry.getValue());
 +      }
 +      bs.close();
 +
 +      long t2 = System.currentTimeMillis();
 +
 +      log.info(String.format("rate : %06.2f%n", ranges.size() / ((t2 - t1) / 1000.0)));
 +
 +      if (!found.equals(expected))
 +        throw new Exception("Found and expected differ " + found + " " + expected);
 +    }
 +
 +    splits = getConnector().tableOperations().listSplits(tableName);
 +    log.info("splits : " + splits);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
index e2dd903,0000000..465936e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
@@@ -1,181 -1,0 +1,182 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +import java.util.concurrent.TimeUnit;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BatchWriterFlushIT extends SimpleMacIT {
 +
 +  private static final int NUM_TO_FLUSH = 100000;
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 90;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    String[] tableNames = getUniqueNames(2);
 +    String bwft = tableNames[0];
 +    c.tableOperations().create(bwft);
 +    String bwlt = tableNames[1];
 +    c.tableOperations().create(bwlt);
 +    runFlushTest(bwft);
 +    runLatencyTest(bwlt);
 +
 +  }
 +
 +  private void runLatencyTest(String tableName) throws Exception {
 +    // should automatically flush after 2 seconds
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(1000, TimeUnit.MILLISECONDS));
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +
 +    Mutation m = new Mutation(new Text(String.format("r_%10d", 1)));
-     m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(Constants.UTF8)));
++    m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +
 +    UtilWaitThread.sleep(500);
 +
 +    int count = 0;
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +
 +    if (count != 0) {
 +      throw new Exception("Flushed too soon");
 +    }
 +
 +    UtilWaitThread.sleep(1500);
 +
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +
 +    if (count != 1) {
 +      throw new Exception("Did not flush");
 +    }
 +
 +    bw.close();
 +  }
 +
 +  private void runFlushTest(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException,
 +      Exception {
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +    Random r = new Random();
 +
 +    for (int i = 0; i < 4; i++) {
 +      for (int j = 0; j < NUM_TO_FLUSH; j++) {
 +        int row = i * NUM_TO_FLUSH + j;
 +
 +        Mutation m = new Mutation(new Text(String.format("r_%10d", row)));
 +        m.put(new Text("cf"), new Text("cq"), new Value(("" + row).getBytes()));
 +        bw.addMutation(m);
 +      }
 +
 +      bw.flush();
 +
 +      // do a few random lookups into the data just flushed
 +
 +      for (int k = 0; k < 10; k++) {
 +        int rowToLookup = r.nextInt(NUM_TO_FLUSH) + i * NUM_TO_FLUSH;
 +
 +        scanner.setRange(new Range(new Text(String.format("r_%10d", rowToLookup))));
 +
 +        Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +        if (!iter.hasNext())
 +          throw new Exception(" row " + rowToLookup + " not found after flush");
 +
 +        Entry<Key,Value> entry = iter.next();
 +
 +        if (iter.hasNext())
 +          throw new Exception("Scanner returned too much");
 +
 +        verifyEntry(rowToLookup, entry);
 +      }
 +
 +      // scan all data just flushed
 +      scanner.setRange(new Range(new Text(String.format("r_%10d", i * NUM_TO_FLUSH)), true, new Text(String.format("r_%10d", (i + 1) * NUM_TO_FLUSH)), false));
 +      Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +      for (int j = 0; j < NUM_TO_FLUSH; j++) {
 +        int row = i * NUM_TO_FLUSH + j;
 +
 +        if (!iter.hasNext())
 +          throw new Exception("Scan stopped permaturely at " + row);
 +
 +        Entry<Key,Value> entry = iter.next();
 +
 +        verifyEntry(row, entry);
 +      }
 +
 +      if (iter.hasNext())
 +        throw new Exception("Scanner returned too much");
 +
 +    }
 +
 +    bw.close();
 +
 +    // test adding a mutation to a closed batch writer
 +    boolean caught = false;
 +    try {
 +      bw.addMutation(new Mutation(new Text("foobar")));
 +    } catch (IllegalStateException ise) {
 +      caught = true;
 +    }
 +
 +    if (!caught) {
 +      throw new Exception("Adding to closed batch writer did not fail");
 +    }
 +  }
 +
 +  private void verifyEntry(int row, Entry<Key,Value> entry) throws Exception {
 +    if (!entry.getKey().getRow().toString().equals(String.format("r_%10d", row))) {
 +      throw new Exception("Unexpected key returned, expected " + row + " got " + entry.getKey());
 +    }
 +
 +    if (!entry.getValue().toString().equals("" + row)) {
 +      throw new Exception("Unexpected value, expected " + row + " got " + entry.getValue());
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 7974c6c,0000000..ffe55bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@@ -1,120 -1,0 +1,121 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVWriter;
 +import org.apache.accumulo.core.file.rfile.RFile;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.trace.TraceFileSystem;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BulkFileIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void testBulkFile() throws Exception {
 +    Connector c = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    c.tableOperations().create(tableName);
 +    SortedSet<Text> splits = new TreeSet<Text>();
 +    for (String split : "0333 0666 0999 1333 1666".split(" "))
 +      splits.add(new Text(split));
 +    c.tableOperations().addSplits(tableName, splits);
 +    Configuration conf = new Configuration();
 +    AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
 +    FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
 +
 +    String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
 +
 +    fs.delete(new Path(dir), true);
 +
 +    FileSKVWriter writer1 = FileOperations.getInstance().openWriter(dir + "/f1." + RFile.EXTENSION, fs, conf, aconf);
 +    writer1.startDefaultLocalityGroup();
 +    writeData(writer1, 0, 333);
 +    writer1.close();
 +
 +    FileSKVWriter writer2 = FileOperations.getInstance().openWriter(dir + "/f2." + RFile.EXTENSION, fs, conf, aconf);
 +    writer2.startDefaultLocalityGroup();
 +    writeData(writer2, 334, 999);
 +    writer2.close();
 +
 +    FileSKVWriter writer3 = FileOperations.getInstance().openWriter(dir + "/f3." + RFile.EXTENSION, fs, conf, aconf);
 +    writer3.startDefaultLocalityGroup();
 +    writeData(writer3, 1000, 1999);
 +    writer3.close();
 +
 +    FunctionalTestUtils.bulkImport(c, fs, tableName, dir);
 +
 +    FunctionalTestUtils.checkRFiles(c, tableName, 6, 6, 1, 1);
 +
 +    verifyData(tableName, 0, 1999);
 +
 +  }
 +
 +  private void verifyData(String table, int s, int e) throws Exception {
 +    Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +    for (int i = s; i <= e; i++) {
 +      if (!iter.hasNext())
 +        throw new Exception("row " + i + " not found");
 +
 +      Entry<Key,Value> entry = iter.next();
 +
 +      String row = String.format("%04d", i);
 +
 +      if (!entry.getKey().getRow().equals(new Text(row)))
 +        throw new Exception("unexpected row " + entry.getKey() + " " + i);
 +
 +      if (Integer.parseInt(entry.getValue().toString()) != i)
 +        throw new Exception("unexpected value " + entry + " " + i);
 +    }
 +
 +    if (iter.hasNext())
 +      throw new Exception("found more than expected " + iter.next());
 +  }
 +
 +  private void writeData(FileSKVWriter w, int s, int e) throws Exception {
 +    for (int i = s; i <= e; i++) {
-       w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(UTF_8)));
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
index 785a997,0000000..92bd714
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
@@@ -1,147 -1,0 +1,148 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.EnumSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ConcurrencyIT extends ConfigurableMacIT {
 +  
 +  static class ScanTask extends Thread {
 +    
 +    int count = 0;
 +    Scanner scanner;
 +    
 +    ScanTask(Connector conn, long time) throws Exception {
 +      scanner = conn.createScanner("cct", Authorizations.EMPTY);
 +      IteratorSetting slow = new IteratorSetting(30, "slow", SlowIterator.class);
 +      SlowIterator.setSleepTime(slow, time);
 +      scanner.addScanIterator(slow);
 +    }
 +    
 +    @Override
 +    public void run() {
 +      count = FunctionalTestUtils.count(scanner);
 +    }
 +    
 +  }
 +  
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1"));
 +  }
 +  
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  /*
 +   * Below is a diagram of the operations in this test over time.
 +   * 
 +   * Scan 0 |------------------------------| Scan 1 |----------| Minc 1 |-----| Scan 2 |----------| Scan 3 |---------------| Minc 2 |-----| Majc 1 |-----|
 +   */
 +  
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    runTest(c);
 +  }
 +
 +  static void runTest(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
 +      MutationsRejectedException, Exception, InterruptedException {
 +    c.tableOperations().create("cct");
 +    IteratorSetting is = new IteratorSetting(10, SlowIterator.class);
 +    SlowIterator.setSleepTime(is, 50);
 +    c.tableOperations().attachIterator("cct", is, EnumSet.of(IteratorScope.minc, IteratorScope.majc));
 +    c.tableOperations().setProperty("cct", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
 +    
 +    BatchWriter bw = c.createBatchWriter("cct", new BatchWriterConfig());
 +    for (int i = 0; i < 50; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%06d", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +    bw.flush();
 +    
 +    ScanTask st0 = new ScanTask(c, 300);
 +    st0.start();
 +    
 +    ScanTask st1 = new ScanTask(c, 100);
 +    st1.start();
 +    
 +    UtilWaitThread.sleep(50);
 +    c.tableOperations().flush("cct", null, null, true);
 +    
 +    for (int i = 0; i < 50; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%06d", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +    
 +    bw.flush();
 +    
 +    ScanTask st2 = new ScanTask(c, 100);
 +    st2.start();
 +    
 +    st1.join();
 +    st2.join();
 +    if (st1.count != 50)
 +      throw new Exception("Thread 1 did not see 50, saw " + st1.count);
 +    
 +    if (st2.count != 50)
 +      throw new Exception("Thread 2 did not see 50, saw " + st2.count);
 +    
 +    ScanTask st3 = new ScanTask(c, 150);
 +    st3.start();
 +    
 +    UtilWaitThread.sleep(50);
 +    c.tableOperations().flush("cct", null, null, false);
 +    
 +    st3.join();
 +    if (st3.count != 50)
 +      throw new Exception("Thread 3 did not see 50, saw " + st3.count);
 +    
 +    st0.join();
 +    if (st0.count != 50)
 +      throw new Exception("Thread 0 did not see 50, saw " + st0.count);
 +    
 +    bw.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
index d5e9188,0000000..7e5944e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@@ -1,317 -1,0 +1,318 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashMap;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.ConstraintViolationSummary;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint;
 +import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ConstraintIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 30;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    String[] tableNames = getUniqueNames(3);
 +    Connector c = getConnector();
 +    for (String table : tableNames) {
 +      c.tableOperations().create(table);
 +      c.tableOperations().addConstraint(table, NumericValueConstraint.class.getName());
 +      c.tableOperations().addConstraint(table, AlphaNumKeyConstraint.class.getName());
 +    }
 +
 +    // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
 +    // logger.setLevel(Level.TRACE);
 +
 +    test1(tableNames[0]);
 +
 +    // logger.setLevel(Level.TRACE);
 +
 +    test2(tableNames[1], false);
 +    test2(tableNames[2], true);
 +  }
 +
 +  private void test1(String tableName) throws Exception {
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    Mutation mut1 = new Mutation(new Text("r1"));
-     mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(Constants.UTF8)));
++    mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(UTF_8)));
 +
 +    bw.addMutation(mut1);
 +
 +    // should not throw any exceptions
 +    bw.close();
 +
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    // create a mutation with a non numeric value
 +    Mutation mut2 = new Mutation(new Text("r1"));
-     mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(Constants.UTF8)));
++    mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(UTF_8)));
 +
 +    bw.addMutation(mut2);
 +
 +    boolean sawMRE = false;
 +
 +    try {
 +      bw.close();
 +      // should not get here
 +      throw new Exception("Test failed, constraint did not catch bad mutation");
 +    } catch (MutationsRejectedException mre) {
 +      sawMRE = true;
 +
 +      // verify constraint violation summary
 +      List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
 +
 +      if (cvsl.size() != 1) {
 +        throw new Exception("Unexpected constraints");
 +      }
 +
 +      for (ConstraintViolationSummary cvs : cvsl) {
 +        if (!cvs.constrainClass.equals(NumericValueConstraint.class.getName())) {
 +          throw new Exception("Unexpected constraint class " + cvs.constrainClass);
 +        }
 +
 +        if (cvs.numberOfViolatingMutations != 1) {
 +          throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
 +        }
 +      }
 +    }
 +
 +    if (!sawMRE) {
 +      throw new Exception("Did not see MutationsRejectedException");
 +    }
 +
 +    // verify mutation did not go through
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new Range(new Text("r1")));
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +    Entry<Key,Value> entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    // remove the numeric value constraint
 +    getConnector().tableOperations().removeConstraint(tableName, 2);
 +    UtilWaitThread.sleep(1000);
 +
 +    // now should be able to add a non numeric value
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +    bw.addMutation(mut2);
 +    bw.close();
 +
 +    // verify mutation went through
 +    iter = scanner.iterator();
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    // add a constraint that references a non-existant class
 +    getConnector().tableOperations().setProperty(tableName, Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistantClass");
 +    UtilWaitThread.sleep(1000);
 +
 +    // add a mutation
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    Mutation mut3 = new Mutation(new Text("r1"));
-     mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++    mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
 +
 +    bw.addMutation(mut3);
 +
 +    sawMRE = false;
 +
 +    try {
 +      bw.close();
 +      // should not get here
 +      throw new Exception("Test failed, mutation went through when table had bad constraints");
 +    } catch (MutationsRejectedException mre) {
 +      sawMRE = true;
 +    }
 +
 +    if (!sawMRE) {
 +      throw new Exception("Did not see MutationsRejectedException");
 +    }
 +
 +    // verify the mutation did not go through
 +    iter = scanner.iterator();
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    // remove the bad constraint
 +    getConnector().tableOperations().removeConstraint(tableName, 1);
 +    UtilWaitThread.sleep(1000);
 +
 +    // try the mutation again
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +    bw.addMutation(mut3);
 +    bw.close();
 +
 +    // verify it went through
 +    iter = scanner.iterator();
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +  }
 +
 +  private Mutation newMut(String row, String cf, String cq, String val) {
 +    Mutation mut1 = new Mutation(new Text(row));
-     mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(Constants.UTF8)));
++    mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(UTF_8)));
 +    return mut1;
 +  }
 +
 +  private void test2(String table, boolean doFlush) throws Exception {
 +    // test sending multiple mutations with multiple constrain violations... all of the non violating mutations
 +    // should go through
 +    int numericErrors = 2;
 +
 +    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
 +    bw.addMutation(newMut("r1", "cf1", "cq1", "123"));
 +    bw.addMutation(newMut("r1", "cf1", "cq2", "I'm a bad value"));
 +    if (doFlush) {
 +      try {
 +        bw.flush();
 +        throw new Exception("Didn't find a bad mutation");
 +      } catch (MutationsRejectedException mre) {
 +        // ignored
 +        try {
 +          bw.close();
 +        } catch (MutationsRejectedException ex) {
 +          // ignored
 +        }
 +        bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
 +        numericErrors = 1;
 +      }
 +    }
 +    bw.addMutation(newMut("r1", "cf1", "cq3", "I'm a naughty value"));
 +    bw.addMutation(newMut("@bad row@", "cf1", "cq2", "456"));
 +    bw.addMutation(newMut("r1", "cf1", "cq4", "789"));
 +
 +    boolean sawMRE = false;
 +
 +    try {
 +      bw.close();
 +      // should not get here
 +      throw new Exception("Test failed, constraint did not catch bad mutation");
 +    } catch (MutationsRejectedException mre) {
 +      System.out.println(mre);
 +
 +      sawMRE = true;
 +
 +      // verify constraint violation summary
 +      List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
 +
 +      if (cvsl.size() != 2) {
 +        throw new Exception("Unexpected constraints");
 +      }
 +
 +      HashMap<String,Integer> expected = new HashMap<String,Integer>();
 +
 +      expected.put("org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", numericErrors);
 +      expected.put("org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint", 1);
 +
 +      for (ConstraintViolationSummary cvs : cvsl) {
 +        if (expected.get(cvs.constrainClass) != cvs.numberOfViolatingMutations) {
 +          throw new Exception("Unexpected " + cvs.constrainClass + " " + cvs.numberOfViolatingMutations);
 +        }
 +      }
 +    }
 +
 +    if (!sawMRE) {
 +      throw new Exception("Did not see MutationsRejectedException");
 +    }
 +
 +    Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +    Entry<Key,Value> entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
index b9b54c7,0000000..5b5249b
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
@@@ -1,128 -1,0 +1,129 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Assert;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +import com.google.common.collect.Iterators;
 +
 +public class CreateAndUseIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 4 * 60;
 +  }
 +
 +  private static SortedSet<Text> splits;
 +
 +  @BeforeClass
 +  public static void createData() throws Exception {
 +    splits = new TreeSet<Text>();
 +
 +    for (int i = 1; i < 256; i++) {
 +      splits.add(new Text(String.format("%08x", i << 8)));
 +    }
 +  }
 +
 +  @Test
 +  public void verifyDataIsPresent() throws Exception {
 +    Text cf = new Text("cf1");
 +    Text cq = new Text("cq1");
 +
 +    String tableName = getUniqueNames(1)[0];
 +    getConnector().tableOperations().create(tableName);
 +    getConnector().tableOperations().addSplits(tableName, splits);
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    for (int i = 1; i < 257; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%08x", (i << 8) - 16)));
-       m.put(cf, cq, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      m.put(cf, cq, new Value(Integer.toString(i).getBytes(UTF_8)));
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +    Scanner scanner1 = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +
 +    int ei = 1;
 +
 +    for (Entry<Key,Value> entry : scanner1) {
 +      Assert.assertEquals(String.format("%08x", (ei << 8) - 16), entry.getKey().getRow().toString());
 +      Assert.assertEquals(Integer.toString(ei), entry.getValue().toString());
 +
 +      ei++;
 +    }
 +
 +    Assert.assertEquals("Did not see expected number of rows", 257, ei);
 +  }
 +
 +  @Test
 +  public void createTableAndScan() throws Exception {
 +    String table2 = getUniqueNames(1)[0];
 +    getConnector().tableOperations().create(table2);
 +    getConnector().tableOperations().addSplits(table2, splits);
 +    Scanner scanner2 = getConnector().createScanner(table2, Authorizations.EMPTY);
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner2) {
 +      if (entry != null)
 +        count++;
 +    }
 +
 +    if (count != 0) {
 +      throw new Exception("Did not see expected number of entries, count = " + count);
 +    }
 +  }
 +
 +  @Test
 +  public void createTableAndBatchScan() throws Exception {
 +    ArrayList<Range> ranges = new ArrayList<Range>();
 +    for (int i = 1; i < 257; i++) {
 +      ranges.add(new Range(new Text(String.format("%08x", (i << 8) - 16))));
 +    }
 +
 +    String table3 = getUniqueNames(1)[0];
 +    getConnector().tableOperations().create(table3);
 +    getConnector().tableOperations().addSplits(table3, splits);
 +    BatchScanner bs = getConnector().createBatchScanner(table3, Authorizations.EMPTY, 3);
 +    bs.setRanges(ranges);
 +    Iterator<Entry<Key,Value>> iter = bs.iterator();
 +    int count = Iterators.size(iter);
 +    bs.close();
 +
 +    Assert.assertEquals("Did not expect to find any entries", 0, count);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
index 83b5b24,0000000..0578ef4
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
@@@ -1,94 -1,0 +1,94 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.*;
 +
 +import java.util.Collections;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class DeleteEverythingIT extends ConfigurableMacIT {
 +  
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1s"));
 +  }
 +  
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    c.tableOperations().create("de");
 +    BatchWriter bw = getConnector().createBatchWriter("de", new BatchWriterConfig());
 +    Mutation m = new Mutation(new Text("foo"));
-     m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(Constants.UTF8)));
++    m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +    bw.flush();
 +    
 +    getConnector().tableOperations().flush("de", null, null, true);
 +    
 +    FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 1, 1);
 +    
 +    m = new Mutation(new Text("foo"));
 +    m.putDelete(new Text("bar"), new Text("1910"));
 +    bw.addMutation(m);
 +    bw.flush();
 +    
 +    Scanner scanner = getConnector().createScanner("de", Authorizations.EMPTY);
 +    scanner.setRange(new Range());
 +    int count = FunctionalTestUtils.count(scanner);
 +    assertEquals("count == " + count, 0, count);
 +    getConnector().tableOperations().flush("de", null, null, true);
 +    
 +    getConnector().tableOperations().setProperty("de", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
 +    UtilWaitThread.sleep(4000);
 +    
 +    FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 0, 0);
 +    
 +    bw.close();
 +    
 +    count = 0;
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +    
 +    if (count != 0)
 +      throw new Exception("count == " + count);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
index a8237a6,0000000..d35ba9f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
@@@ -1,143 -1,0 +1,143 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.junit.Test;
 +
 +// attempt to reproduce ACCUMULO-315
 +public class DeleteRowsSplitIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 4 * 60;
 +  }
 +
 +  private static final Logger log = Logger.getLogger(DeleteRowsSplitIT.class);
 +
 +  private static final String LETTERS = "abcdefghijklmnopqrstuvwxyz";
 +  static final SortedSet<Text> SPLITS = new TreeSet<Text>();
 +  static final List<String> ROWS = new ArrayList<String>();
 +  static {
-     for (byte b : LETTERS.getBytes(Constants.UTF8)) {
++    for (byte b : LETTERS.getBytes(UTF_8)) {
 +      SPLITS.add(new Text(new byte[] {b}));
-       ROWS.add(new String(new byte[] {b}, Constants.UTF8));
++      ROWS.add(new String(new byte[] {b}, UTF_8));
 +    }
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    // Delete ranges of rows, and verify the are removed
 +    // Do this while adding many splits
 +    final String tableName = getUniqueNames(1)[0];
 +
 +    // Eliminate whole tablets
 +    for (int test = 0; test < 10; test++) {
 +      // create a table
 +      log.info("Test " + test);
 +      getConnector().tableOperations().create(tableName);
 +
 +      // put some data in it
 +      fillTable(tableName);
 +
 +      // generate a random delete range
 +      final Text start = new Text();
 +      final Text end = new Text();
 +      generateRandomRange(start, end);
 +
 +      // initiate the delete range
 +      final boolean fail[] = {false};
 +      Thread t = new Thread() {
 +        @Override
 +        public void run() {
 +          try {
 +            // split the table
 +            final SortedSet<Text> afterEnd = SPLITS.tailSet(new Text(end.toString() + "\0"));
 +            getConnector().tableOperations().addSplits(tableName, afterEnd);
 +          } catch (Exception ex) {
 +            log.error(ex, ex);
 +            synchronized (fail) {
 +              fail[0] = true;
 +            }
 +          }
 +        }
 +      };
 +      t.start();
 +
 +      UtilWaitThread.sleep(test * 2);
 +
 +      getConnector().tableOperations().deleteRows(tableName, start, end);
 +
 +      t.join();
 +      synchronized (fail) {
 +        assertTrue(!fail[0]);
 +      }
 +
 +      // scan the table
 +      Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +      for (Entry<Key,Value> entry : scanner) {
 +        Text row = entry.getKey().getRow();
 +        assertTrue(row.compareTo(start) <= 0 || row.compareTo(end) > 0);
 +      }
 +
 +      // delete the table
 +      getConnector().tableOperations().delete(tableName);
 +    }
 +  }
 +
 +  private void generateRandomRange(Text start, Text end) {
 +    List<String> bunch = new ArrayList<String>(ROWS);
 +    Collections.shuffle(bunch);
 +    if (bunch.get(0).compareTo((bunch.get(1))) < 0) {
 +      start.set(bunch.get(0));
 +      end.set(bunch.get(1));
 +    } else {
 +      start.set(bunch.get(1));
 +      end.set(bunch.get(0));
 +    }
 +
 +  }
 +
 +  private void fillTable(String table) throws Exception {
 +    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
 +    for (String row : ROWS) {
 +      Mutation m = new Mutation(row);
 +      m.put("cf", "cq", "value");
 +      bw.addMutation(m);
 +    }
 +    bw.close();
 +  }
 +}


[17/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
index cd26743,0000000..8022c14
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
@@@ -1,210 -1,0 +1,211 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.nio.ByteBuffer;
- import java.nio.charset.Charset;
 +import java.util.List;
 +
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +import static org.easymock.EasyMock.*;
 +import static org.junit.Assert.*;
 +
 +public class ValueTest {
 +  private static final byte[] toBytes(String s) {
-     return s.getBytes(Charset.forName("UTF-8"));
++    return s.getBytes(UTF_8);
 +  }
 +
 +  private static final byte[] DATA = toBytes("data");
 +  private static final ByteBuffer DATABUFF = ByteBuffer.allocate(DATA.length);
 +  static {
 +    DATABUFF.put(DATA);
 +  }
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    DATABUFF.rewind();
 +  }
 +
 +  @Test
 +  public void testDefault() {
 +    Value v = new Value();
 +    assertEquals(0, v.get().length);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullBytesConstructor() {
 +    new Value((byte[]) null);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullCopyConstructor() {
 +    new Value((Value) null);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullByteBufferConstructor() {
 +    new Value((ByteBuffer)null);
 +  }
 +
 +  @Test(expected=NullPointerException.class)
 +  public void testNullSet() {
 +    Value v = new Value();
 +    v.set(null);
 +  }
 +
 +  @Test
 +  public void testByteArray() {
 +    Value v = new Value(DATA);
 +    assertArrayEquals(DATA, v.get());
 +    assertSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteArrayCopy() {
 +    Value v = new Value(DATA, true);
 +    assertArrayEquals(DATA, v.get());
 +    assertNotSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteBuffer() {
 +    Value v = new Value(DATABUFF);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteBufferCopy() {
 +    @SuppressWarnings("deprecation")
 +    Value v = new Value(DATABUFF, true);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testValueCopy() {
 +    Value ov = createMock(Value.class);
 +    expect(ov.get()).andReturn(DATA);
 +    expect(ov.getSize()).andReturn(4);
 +    replay(ov);
 +    Value v = new Value(ov);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testByteArrayOffsetLength() {
 +    Value v = new Value(DATA, 0, 4);
 +    assertArrayEquals(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testSet() {
 +    Value v = new Value();
 +    v.set(DATA);
 +    assertArrayEquals(DATA, v.get());
 +    assertSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testCopy() {
 +    Value v = new Value();
 +    v.copy(DATA);
 +    assertArrayEquals(DATA, v.get());
 +    assertNotSame(DATA, v.get());
 +  }
 +
 +  @Test
 +  public void testGetSize() {
 +    Value v = new Value(DATA);
 +    assertEquals(DATA.length, v.getSize());
 +  }
 +
 +  @Test
 +  public void testGetSizeDefault() {
 +    Value v = new Value();
 +    assertEquals(0, v.getSize());
 +  }
 +
 +  @Test
 +  public void testWriteRead() throws Exception {
 +    Value v = new Value(DATA);
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    DataOutputStream dos = new DataOutputStream(baos);
 +    v.write(dos);
 +    dos.close();
 +    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
 +    DataInputStream dis = new DataInputStream(bais);
 +    Value v2 = new Value();
 +    v2.readFields(dis);
 +    dis.close();
 +    assertArrayEquals(DATA, v2.get());
 +  }
 +
 +  @Test
 +  public void testHashCode() {
 +    Value v1 = new Value(DATA);
 +    Value v2 = new Value(DATA);
 +    assertEquals(v1.hashCode(), v2.hashCode());
 +  }
 +
 +  @Test
 +  public void testCompareTo() {
 +    Value v1 = new Value(DATA);
 +    Value v2 = new Value(toBytes("datb"));
 +    assertTrue(v1.compareTo(v2) < 0);
 +    assertTrue(v2.compareTo(v1) > 0);
 +    Value v1a = new Value(DATA);
 +    assertTrue(v1.compareTo(v1a) == 0);
 +    Value v3 = new Value(toBytes("datc"));
 +    assertTrue(v2.compareTo(v3) < 0);
 +    assertTrue(v1.compareTo(v3) < 0);
 +  }
 +
 +  @Test
 +  public void testEquals() {
 +    Value v1 = new Value(DATA);
 +    assertTrue(v1.equals(v1));
 +    Value v2 = new Value(DATA);
 +    assertTrue(v1.equals(v2));
 +    assertTrue(v2.equals(v1));
 +    Value v3 = new Value(toBytes("datb"));
 +    assertFalse(v1.equals(v3));
 +  }
 +
 +  @Test
 +  public void testToArray() {
 +    List<byte[]> l = new java.util.ArrayList<byte[]>();
 +    byte[] one = toBytes("one");
 +    byte[] two = toBytes("two");
 +    byte[] three = toBytes("three");
 +    l.add(one);
 +    l.add(two);
 +    l.add(three);
 +
 +    byte[][] a = Value.toArray(l);
 +    assertEquals(3, a.length);
 +    assertArrayEquals(one, a[0]);
 +    assertArrayEquals(two, a[1]);
 +    assertArrayEquals(three, a[2]);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index 931ff41,6c4e814..2577e48
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@@ -16,19 -16,11 +16,19 @@@
   */
  package org.apache.accumulo.core.security;
  
++import static com.google.common.base.Charsets.UTF_8;
  import static org.apache.accumulo.core.security.ColumnVisibility.quote;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
  import static org.junit.Assert.fail;
  
 +import java.util.Comparator;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.security.ColumnVisibility.Node;
 +import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator;
 +import org.apache.accumulo.core.security.ColumnVisibility.NodeType;
 +import org.apache.hadoop.io.Text;
  import org.junit.Test;
  
  public class ColumnVisibilityTest {
@@@ -166,83 -152,4 +166,83 @@@
      cv = new ColumnVisibility(quote("五"));
      assertEquals("[\"五\"]", cv.toString());
    }
 +
 +  @Test
 +  public void testParseTree() {
 +    Node node = parse("(W)|(U&V)");
 +    assertNode(node, NodeType.OR, 0, 9);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
 +    assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithNoChildren() {
 +    Node node = parse("ABC");
 +    assertNode(node, NodeType.TERM, 0, 3);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithTwoChildren() {
 +    Node node = parse("ABC|DEF");
 +    assertNode(node, NodeType.OR, 0, 7);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
 +    assertNode(node.getChildren().get(1), NodeType.TERM, 4, 7);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithParenthesesAndTwoChildren() {
 +    Node node = parse("(ABC|DEF)");
 +    assertNode(node, NodeType.OR, 1, 8);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 1, 4);
 +    assertNode(node.getChildren().get(1), NodeType.TERM, 5, 8);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithParenthesizedChildren() {
 +    Node node = parse("ABC|(DEF&GHI)");
 +    assertNode(node, NodeType.OR, 0, 13);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
 +    assertNode(node.getChildren().get(1), NodeType.AND, 5, 12);
 +    assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 8);
 +    assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 9, 12);
 +  }
 +
 +  @Test
 +  public void testParseTreeWithMoreParentheses() {
 +    Node node = parse("(W)|(U&V)");
 +    assertNode(node, NodeType.OR, 0, 9);
 +    assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
 +    assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
 +    assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 6);
 +    assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 7, 8);
 +  }
 +
 +  @Test
 +  public void testEmptyParseTreesAreEqual() {
 +    Comparator<Node> comparator = new NodeComparator(new byte[] {});
 +    Node empty = new ColumnVisibility().getParseTree();
 +    assertEquals(0, comparator.compare(empty, parse("")));
 +  }
 +
 +  @Test
 +  public void testParseTreesOrdering() {
-     byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(Constants.UTF8);
++    byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(UTF_8);
 +    byte[] flattened = new ColumnVisibility(expression).flatten();
 +
 +    // Convert to String for indexOf convenience
-     String flat = new String(flattened, Constants.UTF8);
++    String flat = new String(flattened, UTF_8);
 +    assertTrue("shortest expressions sort first", flat.indexOf('e') < flat.indexOf('|'));
 +    assertTrue("shortest children sort first", flat.indexOf('b') < flat.indexOf('a'));
 +  }
 +
 +  private Node parse(String s) {
 +    ColumnVisibility v = new ColumnVisibility(s);
 +    return v.getParseTree();
 +  }
 +
 +  private void assertNode(Node node, NodeType nodeType, int start, int end) {
 +    assertEquals(node.type, nodeType);
 +    assertEquals(start, node.start);
 +    assertEquals(end, node.end);
 +  }
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
index b925e4a,0000000..a4e9f02
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
@@@ -1,143 -1,0 +1,143 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +import static org.junit.Assert.fail;
 +
 +import javax.security.auth.DestroyFailedException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.client.security.tokens.NullToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.junit.Test;
 +
 +/**
 + * 
 + */
 +public class CredentialsTest {
 +  
 +  @Test
 +  public void testToThrift() throws DestroyFailedException {
 +    // verify thrift serialization
 +    Credentials creds = new Credentials("test", new PasswordToken("testing"));
 +    TCredentials tCreds = creds.toThrift(new MockInstance());
 +    assertEquals("test", tCreds.getPrincipal());
 +    assertEquals(PasswordToken.class.getName(), tCreds.getTokenClassName());
 +    assertArrayEquals(AuthenticationTokenSerializer.serialize(new PasswordToken("testing")), tCreds.getToken());
 +    
 +    // verify that we can't serialize if it's destroyed
 +    creds.getToken().destroy();
 +    try {
 +      creds.toThrift(new MockInstance());
 +      fail();
 +    } catch (Exception e) {
 +      assertTrue(e instanceof RuntimeException);
 +      assertTrue(e.getCause() instanceof AccumuloSecurityException);
 +      assertTrue(AccumuloSecurityException.class.cast(e.getCause()).getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
 +    }
 +  }
 +
 +  @Test
 +  public void roundtripThrift() throws DestroyFailedException {
 +    Credentials creds = new Credentials("test", new PasswordToken("testing"));
 +    TCredentials tCreds = creds.toThrift(new MockInstance());
 +    Credentials roundtrip = Credentials.fromThrift(tCreds);
 +    assertEquals("Roundtrip through thirft changed credentials equality", creds, roundtrip);
 +  }
 +  
 +  @Test
 +  public void testMockConnector() throws AccumuloException, DestroyFailedException, AccumuloSecurityException {
 +    Instance inst = new MockInstance();
 +    Connector rootConnector = inst.getConnector("root", new PasswordToken());
 +    PasswordToken testToken = new PasswordToken("testPass");
 +    rootConnector.securityOperations().createLocalUser("testUser", testToken);
 +    
 +    assertFalse(testToken.isDestroyed());
 +    testToken.destroy();
 +    assertTrue(testToken.isDestroyed());
 +    try {
 +      inst.getConnector("testUser", testToken);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      assertTrue(e.getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
 +    }
 +  }
 +  
 +  @Test
 +  public void testEqualsAndHashCode() {
 +    Credentials nullNullCreds = new Credentials(null, null);
 +    Credentials abcNullCreds = new Credentials("abc", new NullToken());
 +    Credentials cbaNullCreds = new Credentials("cba", new NullToken());
 +    Credentials abcBlahCreds = new Credentials("abc", new PasswordToken("blah"));
 +    
 +    // check hash codes
 +    assertEquals(0, nullNullCreds.hashCode());
 +    assertEquals("abc".hashCode(), abcNullCreds.hashCode());
 +    assertEquals(abcNullCreds.hashCode(), abcBlahCreds.hashCode());
 +    assertFalse(abcNullCreds.hashCode() == cbaNullCreds.hashCode());
 +    
 +    // identity
 +    assertEquals(abcNullCreds, abcNullCreds);
 +    assertEquals(new Credentials("abc", new NullToken()), abcNullCreds);
 +    // equal, but different token constructors
-     assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(Constants.UTF8))), new Credentials("abc", new PasswordToken("abc")));
++    assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(UTF_8))), new Credentials("abc", new PasswordToken("abc")));
 +    // test not equals
 +    assertFalse(nullNullCreds.equals(abcBlahCreds));
 +    assertFalse(nullNullCreds.equals(abcNullCreds));
 +    assertFalse(abcNullCreds.equals(abcBlahCreds));
 +  }
 +  
 +  @Test
 +  public void testCredentialsSerialization() throws AccumuloSecurityException {
-     Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(Constants.UTF8)));
++    Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(UTF_8)));
 +    String serialized = creds.serialize();
 +    Credentials result = Credentials.deserialize(serialized);
 +    assertEquals(creds, result);
 +    assertEquals("a:b-c", result.getPrincipal());
 +    assertEquals(new PasswordToken("d-e-f"), result.getToken());
 +    
 +    Credentials nullNullCreds = new Credentials(null, null);
 +    serialized = nullNullCreds.serialize();
 +    result = Credentials.deserialize(serialized);
 +    assertEquals(null, result.getPrincipal());
 +    assertEquals(null, result.getToken());
 +  }
 +  
 +  @Test
 +  public void testToString() {
 +    Credentials creds = new Credentials(null, null);
 +    assertEquals(Credentials.class.getName() + ":null:null:<hidden>", creds.toString());
 +    creds = new Credentials("", new NullToken());
 +    assertEquals(Credentials.class.getName() + "::" + NullToken.class.getName() + ":<hidden>", creds.toString());
 +    creds = new Credentials("abc", null);
 +    assertEquals(Credentials.class.getName() + ":abc:null:<hidden>", creds.toString());
 +    creds = new Credentials("abc", new PasswordToken(""));
 +    assertEquals(Credentials.class.getName() + ":abc:" + PasswordToken.class.getName() + ":<hidden>", creds.toString());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index d31a788,0000000..523279a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -1,103 -1,0 +1,103 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.createNiceMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +
 +import java.util.Arrays;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.constraints.Constraint.Environment;
 +import org.apache.accumulo.core.data.ArrayByteSequence;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.junit.Before;
 +import org.junit.Ignore;
 +import org.junit.Test;
 +
 +public class VisibilityConstraintTest {
 +
 +  VisibilityConstraint vc;
 +  Environment env;
 +  Mutation mutation;
 +
 +  static final ColumnVisibility good = new ColumnVisibility("good");
 +  static final ColumnVisibility bad = new ColumnVisibility("bad");
 +
 +  static final String D = "don't care";
 +
 +  static final List<Short> ENOAUTH = Arrays.asList((short) 2);
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    vc = new VisibilityConstraint();
 +    mutation = new Mutation("r");
 +
-     ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
++    ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(UTF_8));
 +
 +    AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
 +    expect(ac.contains(bs)).andReturn(true);
 +    replay(ac);
 +
 +    env = createMock(Environment.class);
 +    expect(env.getAuthorizationsContainer()).andReturn(ac);
 +    replay(env);
 +  }
 +
 +  @Test
 +  public void testNoVisibility() {
 +    mutation.put(D, D, D);
 +    assertNull("authorized", vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testVisibilityNoAuth() {
 +    mutation.put(D, D, bad, D);
 +    assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testGoodVisibilityAuth() {
 +    mutation.put(D, D, good, D);
 +    assertNull("authorized", vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testCachedVisibilities() {
 +    mutation.put(D, D, good, "v");
 +    mutation.put(D, D, good, "v2");
 +    assertNull("authorized", vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  public void testMixedVisibilities() {
 +    mutation.put(D, D, bad, D);
 +    mutation.put(D, D, good, D);
 +    assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
 +  }
 +
 +  @Test
 +  @Ignore
 +  public void testMalformedVisibility() {
 +    // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index ea64ac7,0000000..9078ef9
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -1,162 -1,0 +1,162 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security.crypto;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.IOException;
 +import java.util.Arrays;
 +import java.util.Random;
 +
- import org.apache.accumulo.core.Constants;
 +import org.junit.Test;
 +
 +public class BlockedIOStreamTest {
 +  @Test
 +  public void testLargeBlockIO() throws IOException {
 +    writeRead(1024, 2048);
 +  }
 +
 +  private void writeRead(int blockSize, int expectedSize) throws IOException {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, 1);
 +
 +    String contentString = "My Blocked Content String";
-     byte[] content = contentString.getBytes(Constants.UTF8);
++    byte[] content = contentString.getBytes(UTF_8);
 +    blockOut.write(content);
 +    blockOut.flush();
 +
 +    String contentString2 = "My Other Blocked Content String";
-     byte[] content2 = contentString2.getBytes(Constants.UTF8);
++    byte[] content2 = contentString2.getBytes(UTF_8);
 +    blockOut.write(content2);
 +    blockOut.flush();
 +
 +    blockOut.close();
 +    byte[] written = baos.toByteArray();
 +    assertEquals(expectedSize, written.length);
 +
 +    ByteArrayInputStream biis = new ByteArrayInputStream(written);
 +    BlockedInputStream blockIn = new BlockedInputStream(biis, blockSize, blockSize);
 +    DataInputStream dIn = new DataInputStream(blockIn);
 +
 +    dIn.readFully(content, 0, content.length);
-     String readContentString = new String(content, Constants.UTF8);
++    String readContentString = new String(content, UTF_8);
 +
 +    assertEquals(contentString, readContentString);
 +
 +    dIn.readFully(content2, 0, content2.length);
-     String readContentString2 = new String(content2, Constants.UTF8);
++    String readContentString2 = new String(content2, UTF_8);
 +
 +    assertEquals(contentString2, readContentString2);
 +
 +    blockIn.close();
 +  }
 +
 +  @Test
 +  public void testSmallBufferBlockedIO() throws IOException {
 +    writeRead(16, (12 + 4) * (int) (Math.ceil(25.0 / 12) + Math.ceil(31.0 / 12)));
 +  }
 +
 +  @Test
 +  public void testSpillingOverOutputStream() throws IOException {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    // buffer will be size 12
 +    BlockedOutputStream blockOut = new BlockedOutputStream(baos, 16, 16);
 +    Random r = new Random(22);
 +
 +    byte[] undersized = new byte[11];
 +    byte[] perfectSized = new byte[12];
 +    byte[] overSized = new byte[13];
 +    byte[] perfectlyOversized = new byte[13];
 +    byte filler = (byte) r.nextInt();
 +
 +    r.nextBytes(undersized);
 +    r.nextBytes(perfectSized);
 +    r.nextBytes(overSized);
 +    r.nextBytes(perfectlyOversized);
 +
 +    // 1 block
 +    blockOut.write(undersized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    // 2 blocks
 +    blockOut.write(perfectSized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    // 2 blocks
 +    blockOut.write(overSized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    // 3 blocks
 +    blockOut.write(undersized);
 +    blockOut.write(perfectlyOversized);
 +    blockOut.write(filler);
 +    blockOut.flush();
 +
 +    blockOut.close();
 +    assertEquals(16 * 8, baos.toByteArray().length);
 +  }
 +
 +  @Test
 +  public void testGiantWrite() throws IOException {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    int blockSize = 16;
 +    // buffer will be size 12
 +    BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, blockSize);
 +    Random r = new Random(22);
 +
 +    int size = 1024 * 1024 * 128;
 +    byte[] giant = new byte[size];
 +    byte[] pattern = new byte[1024];
 +
 +    r.nextBytes(pattern);
 +
 +    for (int i = 0; i < size / 1024; i++) {
 +      System.arraycopy(pattern, 0, giant, i * 1024, 1024);
 +    }
 +
 +    blockOut.write(giant);
 +    blockOut.flush();
 +
 +    blockOut.close();
 +    baos.close();
 +
 +    int blocks = (int) Math.ceil(size / (blockSize - 4.0));
 +    byte[] byteStream = baos.toByteArray();
 +
 +    assertEquals(blocks * 16, byteStream.length);
 +
 +    DataInputStream blockIn = new DataInputStream(new BlockedInputStream(new ByteArrayInputStream(byteStream), blockSize, blockSize));
 +    Arrays.fill(giant, (byte) 0);
 +    blockIn.readFully(giant, 0, size);
 +    blockIn.close();
 +
 +    for (int i = 0; i < size / 1024; i++) {
 +      byte[] readChunk = new byte[1024];
 +      System.arraycopy(giant, i * 1024, readChunk, 0, 1024);
 +      assertArrayEquals(pattern, readChunk);
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
index 36def05,0000000..a2e4f7a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
@@@ -1,176 -1,0 +1,176 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util.format;
 +
- import static org.apache.accumulo.core.Constants.UTF8;
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.anyObject;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.createNiceMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.expectLastCall;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.util.Collections;
 +import java.util.Map;
 +import java.util.TreeMap;
 +
 +import jline.UnsupportedTerminal;
 +import jline.console.ConsoleReader;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.util.shell.Shell;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class DeleterFormatterTest {
 +  DeleterFormatter formatter;
 +  Map<Key,Value> data;
 +  BatchWriter writer;
 +  BatchWriter exceptionWriter;
 +  Shell shellState;
 +
 +  ByteArrayOutputStream baos;
 +  ConsoleReader reader;
 +
 +  SettableInputStream input;
 +
 +  class SettableInputStream extends InputStream {
 +    ByteArrayInputStream bais;
 +
 +    @Override
 +    public int read() throws IOException {
 +      return bais.read();
 +    }
 +
 +    public void set(String in) {
-       bais = new ByteArrayInputStream(in.getBytes(UTF8));
++      bais = new ByteArrayInputStream(in.getBytes(UTF_8));
 +    }
 +  };
 +
 +  @Before
 +  public void setUp() throws IOException, MutationsRejectedException {
 +    input = new SettableInputStream();
 +    baos = new ByteArrayOutputStream();
 +
 +    MutationsRejectedException mre = createMock(MutationsRejectedException.class);
 +
 +    writer = createNiceMock(BatchWriter.class);
 +    exceptionWriter = createNiceMock(BatchWriter.class);
 +    exceptionWriter.close();
 +    expectLastCall().andThrow(mre);
 +    exceptionWriter.addMutation(anyObject(Mutation.class));
 +    expectLastCall().andThrow(mre);
 +
 +    shellState = createNiceMock(Shell.class);
 +
 +    reader = new ConsoleReader(input, baos, new UnsupportedTerminal());
 +    expect(shellState.getReader()).andReturn(reader).anyTimes();
 +
 +    replay(writer, exceptionWriter, shellState);
 +
 +    data = new TreeMap<Key,Value>();
-     data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF8)));
++    data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF_8)));
 +  }
 +
 +  @Test
 +  public void testEmpty() {
 +    formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true);
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testSingle() throws IOException {
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[DELETED]", " r ", "cf", "cq", "value");
 +  }
 +
 +  @Test
 +  public void testNo() throws IOException {
 +    input.set("no\n");
-     data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[SKIPPED]", " r ", "cf", "cq", "value");
 +
 +    assertTrue(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testNoConfirmation() throws IOException {
 +    input.set("");
-     data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +
 +    verify("[SKIPPED]", " r ", "cf", "cq", "value");
 +
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testYes() throws IOException {
 +    input.set("y\nyes\n");
-     data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++    data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
 +    formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    verify("[DELETED]", " r ", "cf", "cq", "value");
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    verify("[DELETED]", " z ", "v2");
 +  }
 +
 +  @Test
 +  public void testMutationException() {
 +    formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true);
 +
 +    assertTrue(formatter.hasNext());
 +    assertNull(formatter.next());
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  private void verify(String... chunks) throws IOException {
 +    reader.flush();
 +
 +    String output = baos.toString();
 +    for (String chunk : chunks) {
 +      assertTrue(output.contains(chunk));
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 8fc68fd,0000000..eef06c3
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@@ -1,90 -1,0 +1,90 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util.format;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.Arrays;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.TreeMap;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.commons.collections.CollectionUtils;
 +import org.apache.commons.collections.Predicate;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ShardedTableDistributionFormatterTest {
 +  ShardedTableDistributionFormatter formatter;
 +
 +  Map<Key,Value> data;
 +
 +  @Before
 +  public void setUp() {
 +    data = new TreeMap<Key,Value>();
 +    formatter = new ShardedTableDistributionFormatter();
 +  }
 +
 +  @Test
 +  public void testInitialize() {
 +    data.put(new Key(), new Value());
 +    data.put(new Key("r", "~tab"), new Value());
 +    formatter.initialize(data.entrySet(), false);
 +
 +    assertTrue(formatter.hasNext());
 +    formatter.next();
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  @Test
 +  public void testAggregate() {
-     data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(Constants.UTF8)));
-     data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(Constants.UTF8)));
-     data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(Constants.UTF8)));
++    data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(UTF_8)));
++    data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(UTF_8)));
++    data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(UTF_8)));
 +
 +    formatter.initialize(data.entrySet(), false);
 +
 +    String[] resultLines = formatter.next().split("\n");
 +    List<String> results = Arrays.asList(resultLines).subList(2, 4);
 +
 +    assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("NULL", 1)));
 +    assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("19700101", 2)));
 +
 +    assertFalse(formatter.hasNext());
 +  }
 +
 +  private static class AggregateReportChecker implements Predicate {
 +    private String day;
 +    private int count;
 +
 +    AggregateReportChecker(String day, int count) {
 +      this.day = day;
 +      this.count = count;
 +    }
 +
 +    @Override
 +    public boolean evaluate(Object arg) {
 +      String resLine = (String) arg;
 +      return resLine.startsWith(this.day) && resLine.endsWith("" + this.count);
 +    }
 +
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 1f8c32d,e411d22..af31cf2
--- a/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@@ -16,11 -16,10 +16,12 @@@
   */
  package org.apache.accumulo.fate;
  
- import java.nio.charset.Charset;
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
 +import java.util.EnumSet;
 +import java.util.Formatter;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
@@@ -37,34 -35,8 +38,33 @@@ import org.apache.zookeeper.KeeperExcep
   * A utility to administer FATE operations
   */
  public class AdminUtil<T> {
-   private static final Charset UTF8 = Charset.forName("UTF-8");
    
 +  private boolean exitOnError = false;
 +  
 +  /**
 +   * Default constructor
 +   */
 +  public AdminUtil() {
 +    this(true);
 +  }
 +  
 +  /**
 +   * Constructor
 +   * 
 +   * @param exitOnError
 +   *          <code>System.exit(1)</code> on error if true
 +   */
 +  public AdminUtil(boolean exitOnError) {
 +    super();
 +    this.exitOnError = exitOnError;
 +  }
 +  
    public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath) throws KeeperException, InterruptedException {
 +    print(zs, zk, lockPath, new Formatter(System.out), null, null);
 +  }
 +  
 +  public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath, Formatter fmt, Set<Long> filterTxid, EnumSet<TStatus> filterStatus)
 +      throws KeeperException, InterruptedException {
      Map<Long,List<String>> heldLocks = new HashMap<Long,List<String>>();
      Map<Long,List<String>> waitingLocks = new HashMap<Long,List<String>>();
      

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index b6890c6,3ef6b0a..ff35364
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -32,8 -33,7 +33,7 @@@ import org.apache.zookeeper.ZooKeeper
  import org.apache.zookeeper.ZooKeeper.States;
  
  public class ZooSession {
-   private static final Charset UTF8 = Charset.forName("UTF-8");
 -  
 +
    public static class ZooSessionShutdownException extends RuntimeException {
  
      private static final long serialVersionUID = 1L;
@@@ -46,19 -46,18 +46,19 @@@
      public ZooSessionInfo(ZooKeeper zooKeeper, ZooWatcher watcher) {
        this.zooKeeper = zooKeeper;
      }
 -    
 +
      ZooKeeper zooKeeper;
    }
 -  
 +
    private static Map<String,ZooSessionInfo> sessions = new HashMap<String,ZooSessionInfo>();
 -  
 +
    private static String sessionKey(String keepers, int timeout, String scheme, byte[] auth) {
-     return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF8));
+     return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF_8));
    }
 -  
 +
    private static class ZooWatcher implements Watcher {
 -    
 +
 +    @Override
      public void process(WatchedEvent event) {
        if (event.getState() == KeeperState.Expired) {
          log.debug("Session expired, state of current session : " + event.getState());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index d5dc1e9,0000000..8216441
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -1,851 -1,0 +1,853 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.minicluster.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.File;
 +import java.io.FileFilter;
 +import java.io.FileWriter;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.InputStreamReader;
 +import java.net.InetSocketAddress;
 +import java.net.Socket;
 +import java.net.URI;
 +import java.net.URISyntaxException;
 +import java.net.URL;
 +import java.net.URLClassLoader;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Properties;
 +import java.util.Set;
 +import java.util.concurrent.Callable;
 +import java.util.concurrent.ExecutionException;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.FutureTask;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.TimeoutException;
 +
 +import org.apache.accumulo.cluster.AccumuloCluster;
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.impl.MasterClient;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.gc.SimpleGarbageCollector;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.master.state.SetGoalState;
 +import org.apache.accumulo.minicluster.ServerType;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.init.Initialize;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.AccumuloStatus;
 +import org.apache.accumulo.server.util.PortUtils;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
 +import org.apache.accumulo.start.Main;
 +import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.accumulo.tserver.TabletServer;
 +import org.apache.commons.configuration.MapConfiguration;
 +import org.apache.commons.io.FileUtils;
 +import org.apache.commons.vfs2.FileObject;
 +import org.apache.commons.vfs2.impl.VFSClassLoader;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.CommonConfigurationKeys;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.hdfs.DFSConfigKeys;
 +import org.apache.hadoop.hdfs.MiniDFSCluster;
 +import org.apache.thrift.TException;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.data.Stat;
 +import org.apache.zookeeper.server.ZooKeeperServerMain;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.base.Predicate;
 +import com.google.common.collect.Maps;
 +
 +/**
 + * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
 + * code against a real Accumulo instance. Its much more accurate for testing than {@link org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower.
 + *
 + * @since 1.6.0
 + */
 +public class MiniAccumuloClusterImpl implements AccumuloCluster {
 +  private static final Logger log = LoggerFactory.getLogger(MiniAccumuloClusterImpl.class);
 +
 +  public static class LogWriter extends Daemon {
 +    private BufferedReader in;
 +    private BufferedWriter out;
 +
 +    public LogWriter(InputStream stream, File logFile) throws IOException {
 +      this.in = new BufferedReader(new InputStreamReader(stream));
 +      out = new BufferedWriter(new FileWriter(logFile));
 +
 +      SimpleTimer.getInstance().schedule(new Runnable() {
 +        @Override
 +        public void run() {
 +          try {
 +            flush();
 +          } catch (IOException e) {
 +            e.printStackTrace();
 +          }
 +        }
 +      }, 1000, 1000);
 +    }
 +
 +    public synchronized void flush() throws IOException {
 +      if (out != null)
 +        out.flush();
 +    }
 +
 +    @Override
 +    public void run() {
 +      String line;
 +
 +      try {
 +        while ((line = in.readLine()) != null) {
 +          out.append(line);
 +          out.append("\n");
 +        }
 +
 +        synchronized (this) {
 +          out.close();
 +          out = null;
 +          in.close();
 +        }
 +
 +      } catch (IOException e) {}
 +    }
 +  }
 +
 +  private boolean initialized = false;
 +  private Process zooKeeperProcess = null;
 +  private Process masterProcess = null;
 +  private Process gcProcess = null;
 +  private List<Process> tabletServerProcesses = Collections.synchronizedList(new ArrayList<Process>());
 +
 +  private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
 +
 +  private File zooCfgFile;
 +  private String dfsUri;
 +
 +  public List<LogWriter> getLogWriters() {
 +    return logWriters;
 +  }
 +
 +  private List<LogWriter> logWriters = new ArrayList<MiniAccumuloClusterImpl.LogWriter>();
 +
 +  private MiniAccumuloConfigImpl config;
 +  private MiniDFSCluster miniDFS = null;
 +  private List<Process> cleanup = new ArrayList<Process>();
 +
 +  private ExecutorService executor;
 +
 +  public Process exec(Class<?> clazz, String... args) throws IOException {
 +    return exec(clazz, null, args);
 +  }
 +
 +  public Process exec(Class<?> clazz, List<String> jvmArgs, String... args) throws IOException {
 +    ArrayList<String> jvmArgs2 = new ArrayList<String>(1 + (jvmArgs == null ? 0 : jvmArgs.size()));
 +    jvmArgs2.add("-Xmx" + config.getDefaultMemory());
 +    if (jvmArgs != null)
 +      jvmArgs2.addAll(jvmArgs);
 +    Process proc = _exec(clazz, jvmArgs2, args);
 +    cleanup.add(proc);
 +    return proc;
 +  }
 +
 +  private boolean containsSiteFile(File f) {
 +    return f.isDirectory() && f.listFiles(new FileFilter() {
 +
 +      @Override
 +      public boolean accept(File pathname) {
 +        return pathname.getName().endsWith("site.xml");
 +      }
 +    }).length > 0;
 +  }
 +
 +  private void append(StringBuilder classpathBuilder, URL url) throws URISyntaxException {
 +    File file = new File(url.toURI());
 +    // do not include dirs containing hadoop or accumulo site files
 +    if (!containsSiteFile(file))
 +      classpathBuilder.append(File.pathSeparator).append(file.getAbsolutePath());
 +  }
 +
 +  private String getClasspath() throws IOException {
 +
 +    try {
 +      ArrayList<ClassLoader> classloaders = new ArrayList<ClassLoader>();
 +
 +      ClassLoader cl = this.getClass().getClassLoader();
 +
 +      while (cl != null) {
 +        classloaders.add(cl);
 +        cl = cl.getParent();
 +      }
 +
 +      Collections.reverse(classloaders);
 +
 +      StringBuilder classpathBuilder = new StringBuilder();
 +      classpathBuilder.append(config.getConfDir().getAbsolutePath());
 +
 +      if (config.getHadoopConfDir() != null)
 +        classpathBuilder.append(File.pathSeparator).append(config.getHadoopConfDir().getAbsolutePath());
 +
 +      if (config.getClasspathItems() == null) {
 +
 +        // assume 0 is the system classloader and skip it
 +        for (int i = 1; i < classloaders.size(); i++) {
 +          ClassLoader classLoader = classloaders.get(i);
 +
 +          if (classLoader instanceof URLClassLoader) {
 +
 +            URLClassLoader ucl = (URLClassLoader) classLoader;
 +
 +            for (URL u : ucl.getURLs()) {
 +              append(classpathBuilder, u);
 +            }
 +
 +          } else if (classLoader instanceof VFSClassLoader) {
 +
 +            VFSClassLoader vcl = (VFSClassLoader) classLoader;
 +            for (FileObject f : vcl.getFileObjects()) {
 +              append(classpathBuilder, f.getURL());
 +            }
 +          } else {
 +            throw new IllegalArgumentException("Unknown classloader type : " + classLoader.getClass().getName());
 +          }
 +        }
 +      } else {
 +        for (String s : config.getClasspathItems())
 +          classpathBuilder.append(File.pathSeparator).append(s);
 +      }
 +
 +      return classpathBuilder.toString();
 +
 +    } catch (URISyntaxException e) {
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  private Process _exec(Class<?> clazz, List<String> extraJvmOpts, String... args) throws IOException {
 +    String javaHome = System.getProperty("java.home");
 +    String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
 +    String classpath = getClasspath();
 +
 +    String className = clazz.getName();
 +
 +    ArrayList<String> argList = new ArrayList<String>();
 +    argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath));
 +    argList.addAll(extraJvmOpts);
 +    for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) {
 +      argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue()));
 +    }
 +    argList.addAll(Arrays.asList("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-Dapple.awt.UIElement=true", Main.class.getName(),
 +        className));
 +    argList.addAll(Arrays.asList(args));
 +
 +    ProcessBuilder builder = new ProcessBuilder(argList);
 +
 +    builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
 +    builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
 +    builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath());
 +    String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator);
 +    builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath);
 +    builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath);
 +
 +    // if we're running under accumulo.start, we forward these env vars
 +    String env = System.getenv("HADOOP_PREFIX");
 +    if (env != null)
 +      builder.environment().put("HADOOP_PREFIX", env);
 +    env = System.getenv("ZOOKEEPER_HOME");
 +    if (env != null)
 +      builder.environment().put("ZOOKEEPER_HOME", env);
 +    builder.environment().put("ACCUMULO_CONF_DIR", config.getConfDir().getAbsolutePath());
 +    // hadoop-2.2 puts error messages in the logs if this is not set
 +    builder.environment().put("HADOOP_HOME", config.getDir().getAbsolutePath());
 +    if (config.getHadoopConfDir() != null)
 +      builder.environment().put("HADOOP_CONF_DIR", config.getHadoopConfDir().getAbsolutePath());
 +
 +    Process process = builder.start();
 +
 +    LogWriter lw;
 +    lw = new LogWriter(process.getErrorStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
 +    logWriters.add(lw);
 +    lw.start();
 +    lw = new LogWriter(process.getInputStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
 +    logWriters.add(lw);
 +    lw.start();
 +
 +    return process;
 +  }
 +
 +  private Process _exec(Class<?> clazz, ServerType serverType, String... args) throws IOException {
 +
 +    List<String> jvmOpts = new ArrayList<String>();
 +    jvmOpts.add("-Xmx" + config.getMemory(serverType));
 +
 +    if (config.isJDWPEnabled()) {
 +      Integer port = PortUtils.getRandomFreePort();
 +      jvmOpts.addAll(buildRemoteDebugParams(port));
 +      debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
 +    }
 +    return _exec(clazz, jvmOpts, args);
 +  }
 +
 +  /**
 +   *
 +   * @param dir
 +   *          An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
 +   *          and Junit provide methods for creating temporary directories.
 +   * @param rootPassword
 +   *          Initial root password for instance.
 +   */
 +  public MiniAccumuloClusterImpl(File dir, String rootPassword) throws IOException {
 +    this(new MiniAccumuloConfigImpl(dir, rootPassword));
 +  }
 +
 +  /**
 +   * @param config
 +   *          initial configuration
 +   */
 +  @SuppressWarnings("deprecation")
 +  public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException {
 +
 +    this.config = config.initialize();
 +
 +    config.getConfDir().mkdirs();
 +    config.getLogDir().mkdirs();
 +    config.getLibDir().mkdirs();
 +    config.getLibExtDir().mkdirs();
 +
 +    if (!config.useExistingInstance()) {
 +      config.getZooKeeperDir().mkdirs();
 +      config.getWalogDir().mkdirs();
 +      config.getAccumuloDir().mkdirs();
 +    }
 +
 +    if (config.useMiniDFS()) {
 +      File nn = new File(config.getAccumuloDir(), "nn");
 +      nn.mkdirs();
 +      File dn = new File(config.getAccumuloDir(), "dn");
 +      dn.mkdirs();
 +      File dfs = new File(config.getAccumuloDir(), "dfs");
 +      dfs.mkdirs();
 +      Configuration conf = new Configuration();
 +      conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
 +      conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
 +      conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
 +      conf.set("dfs.support.append", "true");
 +      conf.set("dfs.datanode.synconclose", "true");
 +      conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
 +      String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath());
 +      miniDFS = new MiniDFSCluster(conf, 1, true, null);
 +      if (oldTestBuildData == null)
 +        System.clearProperty("test.build.data");
 +      else
 +        System.setProperty("test.build.data", oldTestBuildData);
 +      miniDFS.waitClusterUp();
 +      InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
 +      dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
 +      File coreFile = new File(config.getConfDir(), "core-site.xml");
 +      writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
 +      File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
 +      writeConfig(hdfsFile, conf);
 +
 +      Map<String,String> siteConfig = config.getSiteConfig();
 +      siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
 +      siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
 +      config.setSiteConfig(siteConfig);
 +    } else if (config.useExistingInstance()) {
 +      dfsUri = CachedConfiguration.getInstance().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
 +    } else {
 +      dfsUri = "file://";
 +    }
 +
 +    File clientConfFile = config.getClientConfFile();
 +    // Write only the properties that correspond to ClientConfiguration properties
 +    writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate<Entry<String,String>>() {
 +      @Override
 +      public boolean apply(Entry<String,String> v) {
 +        return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null;
 +      }
 +    }));
 +
 +    File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
 +    writeConfig(siteFile, config.getSiteConfig().entrySet());
 +
 +    if (!config.useExistingInstance()) {
 +      zooCfgFile = new File(config.getConfDir(), "zoo.cfg");
 +      FileWriter fileWriter = new FileWriter(zooCfgFile);
 +
 +      // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
 +      Properties zooCfg = new Properties();
 +      zooCfg.setProperty("tickTime", "2000");
 +      zooCfg.setProperty("initLimit", "10");
 +      zooCfg.setProperty("syncLimit", "5");
 +      zooCfg.setProperty("clientPort", config.getZooKeeperPort() + "");
 +      zooCfg.setProperty("maxClientCnxns", "1000");
 +      zooCfg.setProperty("dataDir", config.getZooKeeperDir().getAbsolutePath());
 +      zooCfg.store(fileWriter, null);
 +
 +      fileWriter.close();
 +    }
 +
 +    // disable audit logging for mini....
 +    InputStream auditStream = this.getClass().getResourceAsStream("/auditLog.xml");
 +
 +    if (auditStream != null) {
 +      FileUtils.copyInputStreamToFile(auditStream, new File(config.getConfDir(), "auditLog.xml"));
 +    }
 +  }
 +
 +  private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings) throws IOException {
 +    FileWriter fileWriter = new FileWriter(file);
 +    fileWriter.append("<configuration>\n");
 +
 +    for (Entry<String,String> entry : settings) {
 +      String value = entry.getValue().replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;");
 +      fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + value + "</value></property>\n");
 +    }
 +    fileWriter.append("</configuration>\n");
 +    fileWriter.close();
 +  }
 +
 +  private void writeConfigProperties(File file, Map<String,String> settings) throws IOException {
 +    FileWriter fileWriter = new FileWriter(file);
 +
 +    for (Entry<String,String> entry : settings.entrySet())
 +      fileWriter.append(entry.getKey() + "=" + entry.getValue() + "\n");
 +    fileWriter.close();
 +  }
 +
 +  /**
 +   * Starts Accumulo and Zookeeper processes. Can only be called once.
 +   *
 +   * @throws IllegalStateException
 +   *           if already started
 +   */
 +  @Override
 +  public synchronized void start() throws IOException, InterruptedException {
 +    if (config.useExistingInstance()) {
 +      Configuration acuConf = config.getAccumuloConfiguration();
 +      Configuration hadoopConf = config.getHadoopConfiguration();
 +
 +      ConfigurationCopy cc = new ConfigurationCopy(acuConf);
 +      VolumeManager fs;
 +      try {
 +        fs = VolumeManagerImpl.get(cc, hadoopConf);
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
 +
 +      String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
 +      IZooReaderWriter zrw = new ZooReaderWriterFactory().getZooReaderWriter(cc.get(Property.INSTANCE_ZK_HOST),
 +          (int) cc.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), cc.get(Property.INSTANCE_SECRET));
 +
 +      String rootPath = ZooUtil.getRoot(instanceIdFromFile);
 +
 +      String instanceName = null;
 +      try {
 +        for (String name : zrw.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
 +          String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
 +          byte[] bytes = zrw.getData(instanceNamePath, new Stat());
-           String iid = new String(bytes, Constants.UTF8);
++          String iid = new String(bytes, UTF_8);
 +          if (iid.equals(instanceIdFromFile)) {
 +            instanceName = name;
 +          }
 +        }
 +      } catch (KeeperException e) {
 +        throw new RuntimeException("Unable to read instance name from zookeeper.", e);
 +      }
 +      if (instanceName == null)
 +        throw new RuntimeException("Unable to read instance name from zookeeper.");
 +
 +      config.setInstanceName(instanceName);
 +      if (!AccumuloStatus.isAccumuloOffline(zrw, rootPath))
 +        throw new RuntimeException("The Accumulo instance being used is already running. Aborting.");
 +    } else {
 +      if (!initialized) {
 +        Runtime.getRuntime().addShutdownHook(new Thread() {
 +          @Override
 +          public void run() {
 +            try {
 +              MiniAccumuloClusterImpl.this.stop();
 +            } catch (IOException e) {
 +              e.printStackTrace();
 +            } catch (InterruptedException e) {
 +              e.printStackTrace();
 +            }
 +          }
 +        });
 +      }
 +
 +      if (zooKeeperProcess == null) {
 +        zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
 +      }
 +
 +      if (!initialized) {
 +        // sleep a little bit to let zookeeper come up before calling init, seems to work better
 +        long startTime = System.currentTimeMillis();
 +        while (true) {
 +          Socket s = null;
 +          try {
 +            s = new Socket("localhost", config.getZooKeeperPort());
 +            s.getOutputStream().write("ruok\n".getBytes());
 +            s.getOutputStream().flush();
 +            byte buffer[] = new byte[100];
 +            int n = s.getInputStream().read(buffer);
 +            if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
 +              break;
 +          } catch (Exception e) {
 +            if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
 +              throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in "
 +                  + config.getLogDir() + " for errors.  Last exception: " + e);
 +            }
 +          } finally {
 +            if (s != null)
 +              s.close();
 +          }
 +        }
 +        Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
 +        int ret = initProcess.waitFor();
 +        if (ret != 0) {
 +          throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
 +        }
 +        initialized = true;
 +      }
 +    }
 +    
 +    log.info("Starting MAC against instance {} and zookeeper(s) {}.", config.getInstanceName(), config.getZooKeepers());
 +    
 +    synchronized (tabletServerProcesses) {
 +      for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
 +        tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
 +      }
 +    }
 +    int ret = 0;
 +    for (int i = 0; i < 5; i++) {
 +      ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()).waitFor();
 +      if (ret == 0)
 +        break;
 +      UtilWaitThread.sleep(1000);
 +    }
 +    if (ret != 0) {
 +      throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
 +    }
 +    if (masterProcess == null) {
 +      masterProcess = _exec(Master.class, ServerType.MASTER);
 +    }
 +
 +    if (gcProcess == null) {
 +      gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
 +    }
 +
 +    if (null == executor) {
 +      executor = Executors.newSingleThreadExecutor();
 +    }
 +  }
 +
 +  private List<String> buildRemoteDebugParams(int port) {
 +    return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d", port)});
 +  }
 +
 +  /**
 +   * @return generated remote debug ports if in debug mode.
 +   * @since 1.6.0
 +   */
 +  public Set<Pair<ServerType,Integer>> getDebugPorts() {
 +    return debugPorts;
 +  }
 +
 +  List<ProcessReference> references(Process... procs) {
 +    List<ProcessReference> result = new ArrayList<ProcessReference>();
 +    for (Process proc : procs) {
 +      result.add(new ProcessReference(proc));
 +    }
 +    return result;
 +  }
 +
 +  public Map<ServerType,Collection<ProcessReference>> getProcesses() {
 +    Map<ServerType,Collection<ProcessReference>> result = new HashMap<ServerType,Collection<ProcessReference>>();
 +    result.put(ServerType.MASTER, references(masterProcess));
 +    result.put(ServerType.TABLET_SERVER, references(tabletServerProcesses.toArray(new Process[0])));
 +    result.put(ServerType.ZOOKEEPER, references(zooKeeperProcess));
 +    if (null != gcProcess) {
 +      result.put(ServerType.GARBAGE_COLLECTOR, references(gcProcess));
 +    }
 +    return result;
 +  }
 +
 +  public void killProcess(ServerType type, ProcessReference proc) throws ProcessNotFoundException, InterruptedException {
 +    boolean found = false;
 +    switch (type) {
 +      case MASTER:
 +        if (proc.equals(masterProcess)) {
 +          masterProcess.destroy();
 +          masterProcess.waitFor();
 +          masterProcess = null;
 +          found = true;
 +        }
 +        break;
 +      case TABLET_SERVER:
 +        synchronized (tabletServerProcesses) {
 +          for (Process tserver : tabletServerProcesses) {
 +            if (proc.equals(tserver)) {
 +              tabletServerProcesses.remove(tserver);
 +              tserver.destroy();
 +              tserver.waitFor();
 +              found = true;
 +              break;
 +            }
 +          }
 +        }
 +        break;
 +      case ZOOKEEPER:
 +        if (proc.equals(zooKeeperProcess)) {
 +          zooKeeperProcess.destroy();
 +          zooKeeperProcess.waitFor();
 +          zooKeeperProcess = null;
 +          found = true;
 +        }
 +        break;
 +      case GARBAGE_COLLECTOR:
 +        if (proc.equals(gcProcess)) {
 +          gcProcess.destroy();
 +          gcProcess.waitFor();
 +          gcProcess = null;
 +          found = true;
 +        }
 +        break;
 +    }
 +    if (!found)
 +      throw new ProcessNotFoundException();
 +  }
 +
 +  /**
 +   * @return Accumulo instance name
 +   */
 +  @Override
 +  public String getInstanceName() {
 +    return config.getInstanceName();
 +  }
 +
 +  /**
 +   * @return zookeeper connection string
 +   */
 +  @Override
 +  public String getZooKeepers() {
 +    return config.getZooKeepers();
 +  }
 +
 +  /**
 +   * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. However its probably best to
 +   * call stop in a finally block as soon as possible.
 +   */
 +  @Override
 +  public synchronized void stop() throws IOException, InterruptedException {
 +    if (null == executor) {
 +      // keep repeated calls to stop() from failing
 +      return;
 +    }
 +
 +    for (LogWriter lw : logWriters) {
 +      lw.flush();
 +    }
 +
 +    if (gcProcess != null) {
 +      try {
 +        stopProcessWithTimeout(gcProcess, 30, TimeUnit.SECONDS);
 +      } catch (ExecutionException e) {
 +        log.warn("GarbageCollector did not fully stop after 30 seconds", e);
 +      } catch (TimeoutException e) {
 +        log.warn("GarbageCollector did not fully stop after 30 seconds", e);
 +      }
 +    }
 +    if (masterProcess != null) {
 +      try {
 +        stopProcessWithTimeout(masterProcess, 30, TimeUnit.SECONDS);
 +      } catch (ExecutionException e) {
 +        log.warn("Master did not fully stop after 30 seconds", e);
 +      } catch (TimeoutException e) {
 +        log.warn("Master did not fully stop after 30 seconds", e);
 +      }
 +    }
 +    if (tabletServerProcesses != null) {
 +      synchronized (tabletServerProcesses) {
 +        for (Process tserver : tabletServerProcesses) {
 +          try {
 +            stopProcessWithTimeout(tserver, 30, TimeUnit.SECONDS);
 +          } catch (ExecutionException e) {
 +            log.warn("TabletServer did not fully stop after 30 seconds", e);
 +          } catch (TimeoutException e) {
 +            log.warn("TabletServer did not fully stop after 30 seconds", e);
 +          }
 +        }
 +      }
 +    }
 +    if (zooKeeperProcess != null) {
 +      try {
 +        stopProcessWithTimeout(zooKeeperProcess, 30, TimeUnit.SECONDS);
 +      } catch (ExecutionException e) {
 +        log.warn("ZooKeeper did not fully stop after 30 seconds", e);
 +      } catch (TimeoutException e) {
 +        log.warn("ZooKeeper did not fully stop after 30 seconds", e);
 +      }
 +    }
 +
 +    zooKeeperProcess = null;
 +    masterProcess = null;
 +    gcProcess = null;
 +    tabletServerProcesses.clear();
 +
 +    // ACCUMULO-2985 stop the ExecutorService after we finished using it to stop accumulo procs
 +    if (null != executor) {
 +      List<Runnable> tasksRemaining = executor.shutdownNow();
 +
 +      // the single thread executor shouldn't have any pending tasks, but check anyways
 +      if (!tasksRemaining.isEmpty()) {
 +        log.warn("Unexpectedly had " + tasksRemaining.size() + " task(s) remaining in threadpool for execution when being stopped");
 +      }
 +
 +      executor = null;
 +    }
 +
 +    if (config.useMiniDFS() && miniDFS != null)
 +      miniDFS.shutdown();
 +    for (Process p : cleanup) {
 +      p.destroy();
 +      p.waitFor();
 +    }
 +    miniDFS = null;
 +  }
 +
 +  /**
 +   * @since 1.6.0
 +   */
 +  @Override
 +  public MiniAccumuloConfigImpl getConfig() {
 +    return config;
 +  }
 +
 +  /**
 +   * Utility method to get a connector to the MAC.
 +   *
 +   * @since 1.6.0
 +   */
 +  @Override
 +  public Connector getConnector(String user, String passwd) throws AccumuloException, AccumuloSecurityException {
 +    Instance instance = new ZooKeeperInstance(getClientConfig());
 +    return instance.getConnector(user, new PasswordToken(passwd));
 +  }
 +
 +  @Override
 +  public ClientConfiguration getClientConfig() {
 +    return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(
 +        this.getZooKeepers());
 +  }
 +
 +  public FileSystem getFileSystem() {
 +    try {
 +      return FileSystem.get(new URI(dfsUri), new Configuration());
 +    } catch (Exception e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  // Visible for testing
 +  protected void setShutdownExecutor(ExecutorService svc) {
 +    this.executor = svc;
 +  }
 +
 +  // Visible for testing
 +  protected ExecutorService getShutdownExecutor() {
 +    return executor;
 +  }
 +
 +  private int stopProcessWithTimeout(final Process proc, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
 +    FutureTask<Integer> future = new FutureTask<Integer>(new Callable<Integer>() {
 +      @Override
 +      public Integer call() throws InterruptedException {
 +        proc.destroy();
 +        return proc.waitFor();
 +      }
 +    });
 +
 +    executor.execute(future);
 +
 +    return future.get(timeout, unit);
 +  }
 +
 +  /**
 +   * Get programmatic interface to information available in a normal monitor. XXX the returned structure won't contain information about the metadata table
 +   * until there is data in it. e.g. if you want to see the metadata table you should create a table.
 +   * 
 +   * @since 1.6.1
 +   */
 +  public MasterMonitorInfo getMasterMonitorInfo() throws AccumuloException, AccumuloSecurityException {
 +    MasterClientService.Iface client = null;
 +    MasterMonitorInfo stats = null;
 +    try {
 +      Instance instance = new ZooKeeperInstance(getClientConfig());
 +      client = MasterClient.getConnectionWithRetry(instance);
 +      stats = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get(instance).toThrift(instance));
 +    } catch (ThriftSecurityException exception) {
 +      throw new AccumuloSecurityException(exception);
 +    } catch (TException exception) {
 +      throw new AccumuloException(exception);
 +    } finally {
 +      if (client != null) {
 +        MasterClient.close(client);
 +      }
 +    }
 +    return stats;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
index be29dbb,48e2906..30da0cba
--- a/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
@@@ -16,6 -16,6 +16,8 @@@
   */
  package org.apache.accumulo.proxy;
  
++import static com.google.common.base.Charsets.UTF_8;
++
  import java.nio.ByteBuffer;
  import java.util.ArrayList;
  import java.util.Collection;
@@@ -194,13 -172,11 +195,13 @@@ public class ProxyServer implements Acc
    }
    
    protected Connector getConnector(ByteBuffer login) throws Exception {
-     String[] pair = new String(login.array(), login.position(), login.remaining(), Constants.UTF8).split(",", 2);
 -    TCredentials user = CredentialHelper.fromByteArray(ByteBufferUtil.toBytes(login));
 -    if (user == null)
 -      throw new org.apache.accumulo.proxy.thrift.AccumuloSecurityException("unknown user");
 -    Connector connector = instance.getConnector(user.getPrincipal(), CredentialHelper.extractToken(user));
 -    return connector;
++    String[] pair = new String(login.array(), login.position(), login.remaining(), UTF_8).split(",", 2);
 +    if (instance.getInstanceID().equals(pair[0])) {
 +      Credentials creds = Credentials.deserialize(pair[1]);
 +      return instance.getConnector(creds.getPrincipal(), creds.getToken());
 +    } else {
 +      throw new org.apache.accumulo.core.client.AccumuloSecurityException(pair[0], org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.INVALID_INSTANCEID);
 +    }
    }
    
    private void handleAccumuloException(AccumuloException e) throws org.apache.accumulo.proxy.thrift.TableNotFoundException,
@@@ -1489,7 -1432,8 +1490,7 @@@
    public ByteBuffer login(String principal, Map<String,String> loginProperties) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException {
      try {
        AuthenticationToken token = getToken(principal, loginProperties);
-       ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(Constants.UTF8));
 -      TCredentials credential = CredentialHelper.create(principal, token, instance.getInstanceID());
 -      ByteBuffer login = ByteBuffer.wrap(CredentialHelper.asByteArray(credential));
++      ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(UTF_8));
        getConnector(login); // check to make sure user exists
        return login;
      } catch (AccumuloSecurityException e) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------


[50/50] [abbrv] git commit: Merge branch '1.6'

Posted by ct...@apache.org.
Merge branch '1.6'


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cfbdef13
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cfbdef13
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cfbdef13

Branch: refs/heads/master
Commit: cfbdef1345388767e81a05a154dac22040ad5283
Parents: 64d1bbe 9b20a9d
Author: Christopher Tubbs <ct...@apache.org>
Authored: Sat Nov 1 00:54:59 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Sat Nov 1 00:54:59 2014 -0400

----------------------------------------------------------------------
 assemble/bin/bootstrap_config.sh                |  2 +-
 .../org/apache/accumulo/core/Constants.java     |  5 ++-
 .../apache/accumulo/core/cli/ClientOpts.java    |  7 ++--
 .../accumulo/core/client/BatchWriterConfig.java | 11 ++---
 .../accumulo/core/client/ZooKeeperInstance.java | 14 +++----
 .../client/impl/InstanceOperationsImpl.java     |  4 +-
 .../accumulo/core/client/impl/Namespaces.java   |  7 ++--
 .../core/client/impl/ReplicationClient.java     |  4 +-
 .../accumulo/core/client/impl/ServerClient.java |  4 +-
 .../core/client/impl/TableOperationsImpl.java   | 32 +++++++--------
 .../accumulo/core/client/impl/Tables.java       | 10 ++---
 .../accumulo/core/client/impl/Writer.java       |  5 +--
 .../core/client/lexicoder/StringLexicoder.java  |  6 +--
 .../client/security/tokens/PasswordToken.java   |  5 ++-
 .../accumulo/core/conf/ConfigurationDocGen.java |  9 ++--
 .../accumulo/core/data/ArrayByteSequence.java   |  7 ++--
 .../org/apache/accumulo/core/data/Column.java   |  7 ++--
 .../apache/accumulo/core/data/Condition.java    |  8 ++--
 .../org/apache/accumulo/core/data/Value.java    |  4 +-
 .../accumulo/core/file/BloomFilterLayer.java    |  7 ++--
 .../core/iterators/ColumnFamilyCounter.java     |  5 ++-
 .../accumulo/core/iterators/LongCombiner.java   |  7 ++--
 .../accumulo/core/iterators/conf/ColumnSet.java |  7 ++--
 .../core/iterators/user/BigDecimalCombiner.java |  7 ++--
 .../core/iterators/user/GrepIterator.java       |  5 ++-
 .../iterators/user/IntersectingIterator.java    |  7 ++--
 .../core/iterators/user/LargeRowFilter.java     |  5 ++-
 .../core/iterators/user/RegExFilter.java        |  7 ++--
 .../iterators/user/RowDeletingIterator.java     |  5 ++-
 .../iterators/user/SummingArrayCombiner.java    |  7 ++--
 .../iterators/user/TransformingIterator.java    |  7 ++--
 .../core/iterators/user/VisibilityFilter.java   |  5 ++-
 .../core/replication/ReplicationSchema.java     |  6 +--
 .../core/replication/ReplicationTarget.java     |  6 +--
 .../accumulo/core/security/Authorizations.java  | 12 +++---
 .../core/security/ColumnVisibility.java         | 43 ++++++++++----------
 .../accumulo/core/security/Credentials.java     | 12 +++---
 .../core/security/VisibilityConstraint.java     |  5 ++-
 .../core/security/VisibilityParseException.java |  5 ++-
 .../accumulo/core/trace/ZooTraceClient.java     |  5 ++-
 .../apache/accumulo/core/util/ByteArraySet.java |  5 ++-
 .../accumulo/core/util/ByteBufferUtil.java      |  5 ++-
 .../apache/accumulo/core/util/CreateToken.java  |  6 +--
 .../org/apache/accumulo/core/util/Encoding.java |  4 +-
 .../apache/accumulo/core/util/FastFormat.java   |  4 +-
 .../org/apache/accumulo/core/util/Merge.java    |  5 ++-
 .../apache/accumulo/core/util/MonitorUtil.java  |  4 +-
 .../org/apache/accumulo/core/util/TextUtil.java |  6 +--
 .../core/client/BatchWriterConfigTest.java      |  8 ++--
 .../core/client/ZooKeeperInstanceTest.java      | 10 ++---
 .../core/client/mock/MockNamespacesTest.java    |  4 +-
 .../tokens/CredentialProviderTokenTest.java     |  8 ++--
 .../security/tokens/PasswordTokenTest.java      |  6 +--
 .../accumulo/core/data/ConditionTest.java       | 13 +++---
 .../core/data/ConditionalMutationTest.java      | 10 ++---
 .../apache/accumulo/core/data/ValueTest.java    |  5 ++-
 .../core/iterators/user/RegExFilterTest.java    |  7 ++--
 .../core/security/ColumnVisibilityTest.java     |  6 +--
 .../accumulo/core/security/CredentialsTest.java |  7 ++--
 .../core/security/VisibilityConstraintTest.java |  4 +-
 .../security/crypto/BlockedIOStreamTest.java    | 10 ++---
 .../ShardedTableDistributionFormatterTest.java  |  8 ++--
 .../org/apache/accumulo/fate/AdminUtil.java     |  8 ++--
 .../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++----
 .../zookeeper/DistributedReadWriteLock.java     | 18 ++++----
 .../accumulo/fate/zookeeper/ZooCache.java       |  8 ++--
 .../apache/accumulo/fate/zookeeper/ZooLock.java |  6 +--
 .../accumulo/fate/zookeeper/ZooReservation.java |  9 ++--
 .../accumulo/fate/zookeeper/ZooSession.java     |  6 +--
 .../core/client/mapreduce/RangeInputSplit.java  |  7 ++--
 .../mapreduce/lib/impl/ConfiguratorBase.java    |  5 +--
 .../mapreduce/lib/impl/InputConfigurator.java   | 16 ++++----
 .../mapreduce/lib/impl/OutputConfigurator.java  |  8 ++--
 .../lib/partition/RangePartitioner.java         |  7 ++--
 .../impl/MiniAccumuloClusterImpl.java           |  5 ++-
 .../org/apache/accumulo/proxy/ProxyServer.java  |  7 ++--
 .../apache/accumulo/proxy/TestProxyClient.java  | 15 +++----
 .../java/org/apache/accumulo/proxy/Util.java    |  5 ++-
 .../org/apache/accumulo/server/Accumulo.java    |  5 ++-
 .../accumulo/server/client/HdfsZooInstance.java | 11 ++---
 .../server/conf/ZooCachePropertyAccessor.java   |  5 ++-
 .../accumulo/server/conf/ZooConfiguration.java  |  5 ++-
 .../server/constraints/MetadataConstraints.java | 15 +++----
 .../apache/accumulo/server/init/Initialize.java | 16 ++++----
 .../accumulo/server/master/LiveTServerSet.java  |  4 +-
 .../server/master/state/DeadServerList.java     |  7 ++--
 .../server/master/state/TServerInstance.java    |  8 ++--
 .../master/state/TabletStateChangeIterator.java |  5 ++-
 .../accumulo/server/master/state/ZooStore.java  |  5 ++-
 .../master/state/ZooTabletStateStore.java       | 11 ++---
 .../server/metrics/AbstractMetricsImpl.java     |  5 ++-
 .../accumulo/server/monitor/LogService.java     |  5 ++-
 .../server/replication/ReplicationUtil.java     |  5 ++-
 .../server/security/SecurityOperation.java      |  5 ++-
 .../server/security/SystemCredentials.java      | 11 ++---
 .../security/handler/ZKAuthenticator.java       |  5 ++-
 .../server/security/handler/ZKAuthorizor.java   |  5 ++-
 .../server/security/handler/ZKPermHandler.java  |  5 ++-
 .../accumulo/server/tables/TableManager.java    | 19 +++++----
 .../server/tablets/UniqueNameAllocator.java     | 11 ++---
 .../server/util/AddFilesWithMissingEntries.java |  5 ++-
 .../accumulo/server/util/ChangeSecret.java      |  5 ++-
 .../accumulo/server/util/CleanZookeeper.java    |  8 ++--
 .../accumulo/server/util/DeleteZooInstance.java |  7 ++--
 .../accumulo/server/util/DumpZookeeper.java     |  6 +--
 .../accumulo/server/util/ListInstances.java     |  7 ++--
 .../server/util/MasterMetadataUtil.java         | 13 +++---
 .../accumulo/server/util/MetadataTableUtil.java | 33 +++++++--------
 .../accumulo/server/util/NamespacePropUtil.java |  4 +-
 .../accumulo/server/util/RandomizeVolumes.java  |  4 +-
 .../accumulo/server/util/RestoreZookeeper.java  |  7 ++--
 .../accumulo/server/util/SendLogToChainsaw.java |  9 ++--
 .../accumulo/server/util/SystemPropUtil.java    |  4 +-
 .../accumulo/server/util/TablePropUtil.java     |  4 +-
 .../accumulo/server/util/TabletServerLocks.java |  5 ++-
 .../server/watcher/MonitorLog4jWatcher.java     |  4 +-
 .../server/zookeeper/DistributedWorkQueue.java  |  7 ++--
 .../accumulo/server/zookeeper/ZooQueueLock.java |  9 ++--
 .../server/zookeeper/ZooReaderWriter.java       |  4 +-
 .../zookeeper/ZooReaderWriterFactory.java       |  4 +-
 .../server/conf/NamespaceConfigurationTest.java | 31 +++++++-------
 .../conf/ServerConfigurationFactoryTest.java    |  5 +--
 .../server/conf/TableConfigurationTest.java     | 10 ++---
 .../conf/ZooCachePropertyAccessorTest.java      |  4 +-
 .../java/org/apache/accumulo/master/Master.java | 15 +++----
 .../master/recovery/RecoveryManager.java        |  5 ++-
 .../MasterReplicationCoordinator.java           |  5 ++-
 .../accumulo/master/state/SetGoalState.java     |  4 +-
 .../accumulo/master/tableOps/BulkImport.java    |  9 ++--
 .../master/tableOps/CancelCompactions.java      | 10 ++---
 .../accumulo/master/tableOps/CompactRange.java  | 15 +++----
 .../accumulo/master/tableOps/ExportTable.java   |  9 ++--
 .../accumulo/master/tableOps/ImportTable.java   | 17 ++++----
 .../accumulo/master/tableOps/RenameTable.java   |  6 +--
 .../apache/accumulo/master/tableOps/Utils.java  | 15 +++----
 .../master/tserverOps/ShutdownTServer.java      |  4 +-
 .../org/apache/accumulo/monitor/Monitor.java    |  7 ++--
 .../accumulo/monitor/ZooKeeperStatus.java       |  7 ++--
 .../accumulo/monitor/servlets/BasicServlet.java | 11 ++---
 .../monitor/servlets/DefaultServlet.java        |  5 ++-
 .../accumulo/monitor/servlets/ShellServlet.java |  5 ++-
 .../accumulo/monitor/servlets/XMLServlet.java   |  7 ++--
 .../accumulo/monitor/servlets/trace/Basic.java  |  5 ++-
 .../org/apache/accumulo/tracer/TraceServer.java |  9 ++--
 .../tserver/BulkFailedCopyProcessor.java        |  6 +--
 .../apache/accumulo/tserver/TabletServer.java   | 12 +++---
 .../apache/accumulo/tserver/log/DfsLogger.java  | 16 ++++----
 .../accumulo/tserver/logger/LogFileValue.java   | 11 ++---
 .../accumulo/tserver/logger/LogReader.java      |  5 ++-
 .../replication/ReplicationProcessor.java       |  5 ++-
 .../apache/accumulo/tserver/tablet/Tablet.java  | 11 ++---
 .../replication/ReplicationProcessorTest.java   |  5 ++-
 .../java/org/apache/accumulo/shell/Shell.java   | 10 +++--
 .../org/apache/accumulo/shell/ShellUtil.java    |  7 ++--
 .../shell/commands/AuthenticateCommand.java     |  5 ++-
 .../shell/commands/ExecfileCommand.java         |  5 ++-
 .../accumulo/shell/commands/HiddenCommand.java  |  5 ++-
 .../accumulo/shell/commands/PasswdCommand.java  |  6 +--
 .../shell/commands/QuotedStringTokenizer.java   |  5 ++-
 .../accumulo/shell/commands/UserCommand.java    |  5 ++-
 .../apache/accumulo/shell/mock/MockShell.java   |  8 ++--
 .../apache/accumulo/shell/ShellUtilTest.java    |  4 +-
 .../shell/format/DeleterFormatterTest.java      | 12 +++---
 .../accumulo/test/BulkImportDirectory.java      |  5 ++-
 .../accumulo/test/NativeMapConcurrencyTest.java |  5 ++-
 .../accumulo/test/NativeMapPerformanceTest.java |  7 ++--
 .../accumulo/test/NativeMapStressTest.java      |  5 ++-
 .../apache/accumulo/test/TestBinaryRows.java    |  7 ++--
 .../org/apache/accumulo/test/TestIngest.java    |  7 ++--
 .../accumulo/test/TestMultiTableIngest.java     |  5 ++-
 .../test/continuous/ContinuousIngest.java       |  9 ++--
 .../test/continuous/ContinuousMoru.java         |  7 ++--
 .../test/continuous/ContinuousQuery.java        |  5 ++-
 .../test/continuous/ContinuousScanner.java      |  5 ++-
 .../test/continuous/ContinuousVerify.java       |  7 ++--
 .../test/continuous/ContinuousWalk.java         | 17 ++++----
 .../accumulo/test/continuous/Histogram.java     |  5 ++-
 .../test/continuous/PrintScanTimeHistogram.java |  5 ++-
 .../accumulo/test/continuous/TimeBinner.java    |  5 ++-
 .../test/continuous/UndefinedAnalyzer.java      |  9 ++--
 .../test/functional/CacheTestReader.java        |  7 ++--
 .../test/functional/CacheTestWriter.java        | 11 ++---
 .../accumulo/test/functional/ZombieTServer.java |  5 ++-
 .../metadata/MetadataBatchScanTest.java         |  7 ++--
 .../apache/accumulo/test/randomwalk/Module.java |  7 ++--
 .../test/randomwalk/bulk/BulkMinusOne.java      |  4 +-
 .../accumulo/test/randomwalk/bulk/Verify.java   |  5 ++-
 .../test/randomwalk/concurrent/BatchWrite.java  |  5 ++-
 .../test/randomwalk/concurrent/BulkImport.java  |  5 ++-
 .../concurrent/ChangeAuthorizations.java        |  7 ++--
 .../randomwalk/concurrent/StopTabletServer.java |  5 ++-
 .../accumulo/test/randomwalk/image/Verify.java  |  5 ++-
 .../accumulo/test/randomwalk/image/Write.java   |  9 ++--
 .../test/randomwalk/multitable/Write.java       |  5 ++-
 .../randomwalk/security/SecurityHelper.java     |  7 ++--
 .../test/randomwalk/security/TableOp.java       |  9 ++--
 .../test/randomwalk/sequential/Write.java       |  5 ++-
 .../test/randomwalk/shard/BulkInsert.java       |  5 ++-
 .../test/randomwalk/shard/ExportIndex.java      |  5 ++-
 .../accumulo/test/scalability/Ingest.java       |  5 ++-
 .../apache/accumulo/test/AuditMessageIT.java    |  6 +--
 .../apache/accumulo/test/LargeSplitRowIT.java   |  7 ++--
 .../org/apache/accumulo/test/NamespacesIT.java  |  8 ++--
 .../java/org/apache/accumulo/test/VolumeIT.java |  6 +--
 .../accumulo/test/functional/AddSplitIT.java    |  5 ++-
 .../test/functional/BadIteratorMincIT.java      |  6 +--
 .../test/functional/BatchScanSplitIT.java       |  7 ++--
 .../test/functional/BatchWriterFlushIT.java     |  5 ++-
 .../accumulo/test/functional/BulkFileIT.java    |  5 ++-
 .../accumulo/test/functional/ConcurrencyIT.java |  7 ++--
 .../accumulo/test/functional/ConstraintIT.java  | 23 ++++++-----
 .../test/functional/CreateAndUseIT.java         |  5 ++-
 .../test/functional/DeleteEverythingIT.java     |  4 +-
 .../test/functional/DeleteRowsSplitIT.java      |  6 +--
 .../accumulo/test/functional/LargeRowIT.java    |  7 ++--
 .../accumulo/test/functional/NativeMapIT.java   | 16 ++++----
 .../test/functional/ScanIteratorIT.java         |  7 ++--
 .../accumulo/test/functional/ScanRangeIT.java   |  5 ++-
 .../test/functional/ScanSessionTimeOutIT.java   |  5 ++-
 .../test/functional/SplitRecoveryIT.java        |  4 +-
 .../accumulo/test/functional/TabletIT.java      |  4 +-
 .../accumulo/test/functional/VisibilityIT.java  | 11 ++---
 .../replication/MultiTserverReplicationIT.java  |  7 ++--
 .../UnusedWalDoesntCloseReplicationStatus.java  |  6 +--
 .../instrument/receivers/ZooSpanClient.java     |  6 +--
 225 files changed, 928 insertions(+), 790 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/assemble/bin/bootstrap_config.sh
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index e7162c2,9897262..0229d4e
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -16,7 -16,9 +16,8 @@@
   */
  package org.apache.accumulo.core;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
 -import java.nio.charset.Charset;
  import java.util.Arrays;
  import java.util.Collection;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
index e582160,041c9e6..10a5e08
--- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.cli;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.LinkedHashMap;
  import java.util.Map;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
index 23ab39e,24ad86d..842a713
--- a/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.client;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.DataInput;
  import java.io.DataOutput;
  import java.io.IOException;
@@@ -203,12 -180,10 +204,12 @@@ public class BatchWriterConfig implemen
        addField(fields, "maxWriteThreads", maxWriteThreads);
      if (timeout != null)
        addField(fields, "timeout", timeout);
 +    if (durability != Durability.DEFAULT)
 +      addField(fields, "durability", durability);
      String output = StringUtils.join(",", fields);
  
-     byte[] bytes = output.getBytes(StandardCharsets.UTF_8);
-     byte[] len = String.format("%6s#", Integer.toString(bytes.length, 36)).getBytes(StandardCharsets.UTF_8);
+     byte[] bytes = output.getBytes(UTF_8);
+     byte[] len = String.format("%6s#", Integer.toString(bytes.length, 36)).getBytes(UTF_8);
      if (len.length != 7)
        throw new IllegalStateException("encoded length does not match expected value");
      out.write(len);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 3dba358,a9c3292..4e11a14
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -16,10 -16,9 +16,10 @@@
   */
  package org.apache.accumulo.core.client;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.Collections;
  import java.util.List;
  import java.util.UUID;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
index 1a9af63,6925f18..30c15ed
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
@@@ -16,9 -16,8 +16,9 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
index 6be1a12,f32940c..3c2bc45
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.security.SecurityPermission;
  import java.util.LinkedList;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
index 937dc7c,0000000..901128b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
@@@ -1,246 -1,0 +1,246 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
 +import static com.google.common.base.Preconditions.checkNotNull;
++import static java.nio.charset.StandardCharsets.UTF_8;
 +
- import java.nio.charset.StandardCharsets;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
 +import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.thrift.TServiceClient;
 +import org.apache.thrift.transport.TTransportException;
 +import org.apache.zookeeper.KeeperException;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class ReplicationClient {
 +  private static final Logger log = LoggerFactory.getLogger(ReplicationClient.class);
 +
 +  /**
 +   * @param instance
 +   *          Instance for the peer replicant
 +   * @return Client to the ReplicationCoordinator service
 +   */
 +  public static ReplicationCoordinator.Client getCoordinatorConnectionWithRetry(Instance instance) throws AccumuloException {
 +    checkNotNull(instance);
 +
 +    for (int attempts = 1; attempts <= 10; attempts++) {
 +
 +      ReplicationCoordinator.Client result = getCoordinatorConnection(instance);
 +      if (result != null)
 +        return result;
 +      log.debug("Could not get ReplicationCoordinator connection to {}, will retry", instance.getInstanceName());
 +      try {
 +        Thread.sleep(attempts * 250);
 +      } catch (InterruptedException e) {
 +        throw new AccumuloException(e);
 +      }
 +    }
 +
 +    throw new AccumuloException("Timed out trying to communicate with master from " + instance.getInstanceName());
 +  }
 +
 +  public static ReplicationCoordinator.Client getCoordinatorConnection(Instance instance) {
 +    List<String> locations = instance.getMasterLocations();
 +
 +    if (locations.size() == 0) {
 +      log.debug("No masters for replication to instance {}", instance.getInstanceName());
 +      return null;
 +    }
 +
 +    // This is the master thrift service, we just want the hostname, not the port
 +    String masterThriftService = locations.get(0);
 +    if (masterThriftService.endsWith(":0")) {
 +      log.warn("Master found for {} did not have real location {}", instance.getInstanceName(), masterThriftService);
 +      return null;
 +    }
 +
 +
 +    AccumuloConfiguration conf = ClientConfigurationHelper.getClientRpcConfiguration(instance);
 +
 +    String zkPath = ZooUtil.getRoot(instance) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR;
 +    String replCoordinatorAddr;
 +
 +    log.debug("Using ZooKeeper quorum at {} with path {} to find peer Master information", instance.getZooKeepers(), zkPath);
 +
 +    // Get the coordinator port for the master we're trying to connect to
 +    try {
 +      ZooReader reader = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
-       replCoordinatorAddr = new String(reader.getData(zkPath, null), StandardCharsets.UTF_8);
++      replCoordinatorAddr = new String(reader.getData(zkPath, null), UTF_8);
 +    } catch (KeeperException | InterruptedException e) {
 +      log.error("Could not fetch remote coordinator port", e);
 +      return null;
 +    }
 +
 +    // Throw the hostname and port through HostAndPort to get some normalization
 +    HostAndPort coordinatorAddr = HostAndPort.fromString(replCoordinatorAddr);
 +
 +    log.debug("Connecting to master at {}", coordinatorAddr);
 +
 +    try {
 +      // Master requests can take a long time: don't ever time out
 +      ReplicationCoordinator.Client client = ThriftUtil.getClientNoTimeout(new ReplicationCoordinator.Client.Factory(), coordinatorAddr.toString(),
 +          conf);
 +      return client;
 +    } catch (TTransportException tte) {
 +      log.debug("Failed to connect to master coordinator service ({})", coordinatorAddr, tte);
 +      return null;
 +    }
 +  }
 +
 +  /**
 +   * Attempt a single time to create a ReplicationServicer client to the given host
 +   * 
 +   * @param inst
 +   *          Instance to the peer replicant
 +   * @param server
 +   *          Server to connect to
 +   * @return A ReplicationServicer client to the given host in the given instance
 +   */
 +  public static ReplicationServicer.Client getServicerConnection(Instance inst, String server) throws TTransportException {
 +    checkNotNull(inst);
 +    checkNotNull(server);
 +
 +    try {
 +      return ThriftUtil.getClientNoTimeout(new ReplicationServicer.Client.Factory(), server, ClientConfigurationHelper.getClientRpcConfiguration(inst));
 +    } catch (TTransportException tte) {
 +      log.debug("Failed to connect to servicer ({}), will retry...", server, tte);
 +      throw tte;
 +    }
 +  }
 +
 +  public static void close(ReplicationCoordinator.Iface iface) {
 +    TServiceClient client = (TServiceClient) iface;
 +    if (client != null && client.getInputProtocol() != null && client.getInputProtocol().getTransport() != null) {
 +      ThriftTransportPool.getInstance().returnTransport(client.getInputProtocol().getTransport());
 +    } else {
 +      log.debug("Attempt to close null connection to the remote system", new Exception());
 +    }
 +  }
 +
 +  public static void close(ReplicationServicer.Iface iface) {
 +    TServiceClient client = (TServiceClient) iface;
 +    if (client != null && client.getInputProtocol() != null && client.getInputProtocol().getTransport() != null) {
 +      ThriftTransportPool.getInstance().returnTransport(client.getInputProtocol().getTransport());
 +    } else {
 +      log.debug("Attempt to close null connection to the remote system", new Exception());
 +    }
 +  }
 +
 +  public static <T> T executeCoordinatorWithReturn(Instance instance, ClientExecReturn<T,ReplicationCoordinator.Client> exec) throws AccumuloException,
 +      AccumuloSecurityException {
 +    ReplicationCoordinator.Client client = null;
 +    for (int i = 0; i < 10; i++) {
 +      try {
 +        client = getCoordinatorConnectionWithRetry(instance);
 +        return exec.execute(client);
 +      } catch (TTransportException tte) {
 +        log.debug("ReplicationClient coordinator request failed, retrying ... ", tte);
 +        try {
 +          Thread.sleep(100);
 +        } catch (InterruptedException e) {
 +          throw new AccumuloException(e);
 +        }
 +      } catch (ThriftSecurityException e) {
 +        throw new AccumuloSecurityException(e.user, e.code, e);
 +      } catch (AccumuloException e) {
 +        throw e;
 +      } catch (Exception e) {
 +        throw new AccumuloException(e);
 +      } finally {
 +        if (client != null)
 +          close(client);
 +      }
 +    }
 +
 +    throw new AccumuloException("Could not connect to ReplicationCoordinator at " + instance.getInstanceName());
 +  }
 +
 +  public static void executeCoordinator(Instance instance, ClientExec<ReplicationCoordinator.Client> exec) throws AccumuloException, AccumuloSecurityException {
 +    ReplicationCoordinator.Client client = null;
 +    try {
 +      client = getCoordinatorConnectionWithRetry(instance);
 +      exec.execute(client);
 +    } catch (TTransportException tte) {
 +      log.debug("ReplicationClient coordinator request failed, retrying ... ", tte);
 +      UtilWaitThread.sleep(100);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (AccumuloException e) {
 +      throw e;
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (client != null)
 +        close(client);
 +    }
 +  }
 +
 +  public static <T> T executeServicerWithReturn(Instance instance, String tserver, ClientExecReturn<T,ReplicationServicer.Client> exec)
 +      throws AccumuloException, AccumuloSecurityException, TTransportException {
 +    ReplicationServicer.Client client = null;
 +    while (true) {
 +      try {
 +        client = getServicerConnection(instance, tserver);
 +        return exec.execute(client);
 +      } catch (ThriftSecurityException e) {
 +        throw new AccumuloSecurityException(e.user, e.code, e);
 +      } catch (AccumuloException e) {
 +        throw e;
 +      } catch (Exception e) {
 +        throw new AccumuloException(e);
 +      } finally {
 +        if (client != null)
 +          close(client);
 +      }
 +    }
 +  }
 +
 +  public static void executeServicer(Instance instance, String tserver, ClientExec<ReplicationServicer.Client> exec) throws AccumuloException,
 +      AccumuloSecurityException, TTransportException {
 +    ReplicationServicer.Client client = null;
 +    try {
 +      client = getServicerConnection(instance, tserver);
 +      exec.execute(client);
 +      return;
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (AccumuloException e) {
 +      throw e;
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (client != null)
 +        close(client);
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
index fcbb104,70bdbb0..2d76695
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
@@@ -16,9 -16,8 +16,9 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  
  import org.apache.accumulo.core.Constants;
@@@ -130,9 -130,10 +130,9 @@@ public class ServerClient 
      for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
        String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
        byte[] data = ZooUtil.getLockData(zc, path);
-       if (data != null && !new String(data, StandardCharsets.UTF_8).equals("master"))
+       if (data != null && !new String(data, UTF_8).equals("master"))
 -        servers.add(new ThriftTransportKey(
 -          new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT),
 -          rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance))));
 +        servers.add(new ThriftTransportKey(new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT), rpcTimeout, SslConnectionParams
 +            .forClient(ClientConfigurationHelper.getClientRpcConfiguration(instance))));
      }
      
      boolean opened = false;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 4e3d9c6,547ccc7..06b9e15
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@@ -16,7 -16,7 +16,8 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import java.io.BufferedReader;
  import java.io.IOException;
@@@ -212,10 -208,9 +212,10 @@@ public class TableOperationsImpl extend
     */
    @Override
    public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 -    ArgumentChecker.notNull(tableName, timeType);
 +    checkArgument(tableName != null, "tableName is null");
 +    checkArgument(timeType != null, "timeType is null");
  
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(StandardCharsets.UTF_8)));
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(UTF_8)));
  
      Map<String,String> opts;
      if (limitVersion)
@@@ -537,9 -532,9 +537,9 @@@
    @Override
    public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
  
 -    ArgumentChecker.notNull(tableName);
 +    checkArgument(tableName != null, "tableName is null");
      ByteBuffer EMPTY = ByteBuffer.allocate(0);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
          end == null ? EMPTY : TextUtil.getByteBuffer(end));
      Map<String,String> opts = new HashMap<String,String>();
      try {
@@@ -553,9 -548,9 +553,9 @@@
    @Override
    public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
  
 -    ArgumentChecker.notNull(tableName);
 +    checkArgument(tableName != null, "tableName is null");
      ByteBuffer EMPTY = ByteBuffer.allocate(0);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
          end == null ? EMPTY : TextUtil.getByteBuffer(end));
      Map<String,String> opts = new HashMap<String,String>();
      try {
@@@ -677,9 -672,9 +677,9 @@@
     */
    @Override
    public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 -    ArgumentChecker.notNull(tableName);
 +    checkArgument(tableName != null, "tableName is null");
  
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)));
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)));
      Map<String,String> opts = new HashMap<String,String>();
  
      try {
@@@ -1336,9 -1324,9 +1336,9 @@@
    @Override
    public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
  
 -    ArgumentChecker.notNull(tableName);
 +    checkArgument(tableName != null, "tableName is null");
      String tableId = Tables.getTableId(instance, tableName);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(StandardCharsets.UTF_8)));
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
      Map<String,String> opts = new HashMap<String,String>();
  
      try {
@@@ -1368,9 -1356,9 +1368,9 @@@
     */
    @Override
    public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 -    ArgumentChecker.notNull(tableName);
 +    checkArgument(tableName != null, "tableName is null");
      String tableId = Tables.getTableId(instance, tableName);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(StandardCharsets.UTF_8)));
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
      Map<String,String> opts = new HashMap<String,String>();
  
      try {
@@@ -1535,10 -1521,9 +1535,10 @@@
  
    @Override
    public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 -    ArgumentChecker.notNull(tableName, exportDir);
 +    checkArgument(tableName != null, "tableName is null");
 +    checkArgument(exportDir != null, "exportDir is null");
  
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(exportDir.getBytes(StandardCharsets.UTF_8)));
+     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
  
      Map<String,String> opts = Collections.emptyMap();
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
index 0b522ca,4de95c3..c2b8001
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
@@@ -16,9 -16,8 +16,9 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
- import java.nio.charset.StandardCharsets;
  import java.security.SecurityPermission;
  import java.util.HashMap;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index e76e2d9,de7bcb2..a38822b
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@@ -16,9 -16,7 +16,8 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
- 
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.client.AccumuloException;
  import org.apache.accumulo.core.client.AccumuloSecurityException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
index b5694f2,bf365ef..165248b
--- a/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.client.lexicoder;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  /**
   * This lexicoder encodes/decodes a given String to/from bytes without further processing. It can be combined with other encoders like the

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index 39cf8a0,4d73979..53a0bcf
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.client.security.tokens;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.DataInput;
  import java.io.DataOutput;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 51fef69,618ff7e..60d1996
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.conf;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.FileNotFoundException;
  import java.io.IOException;
  import java.io.InputStream;
@@@ -324,9 -311,9 +325,9 @@@ class ConfigurationDocGen 
        int n;
        try {
          while ((n = data.read(buffer)) > 0)
-           doc.print(new String(buffer, 0, n, StandardCharsets.UTF_8));
+           doc.print(new String(buffer, 0, n, UTF_8));
        } catch (IOException e) {
 -        e.printStackTrace();
 +        log.debug("Encountered IOException while reading InputStream in appendResource().", e);
          return;
        } finally {
          try {
@@@ -363,11 -346,11 +364,11 @@@
     */
    public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
      if (args.length == 2 && args[0].equals("--generate-html")) {
-       new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateHtml();
+       new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateHtml();
 -    } else if (args.length == 2 && args[0].equals("--generate-latex")) {
 -      new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateLaTeX();
 +    } else if (args.length == 2 && args[0].equals("--generate-asciidoc")) {
-       new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateAsciidoc();
++      new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateAsciidoc();
      } else {
 -      throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
 +      throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-asciidoc <filename>");
      }
    }
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index 2467585,437590f..954fef3
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@@ -16,13 -16,11 +16,14 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.Serializable;
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  
 +/**
 + * An implementation of {@link ByteSequence} that uses a backing byte array.
 + */
  public class ArrayByteSequence extends ByteSequence implements Serializable {
    
    private static final long serialVersionUID = 1L;
@@@ -67,25 -47,10 +68,25 @@@
      
    }
    
 +  /**
 +   * Creates a new sequence from the given string. The bytes are determined from
 +   * the string using the default platform encoding.
 +   *
 +   * @param s string to represent as bytes
 +   */
    public ArrayByteSequence(String s) {
-     this(s.getBytes(StandardCharsets.UTF_8));
+     this(s.getBytes(UTF_8));
    }
    
 +  /**
 +   * Creates a new sequence based on a byte buffer. If the byte buffer has an
 +   * array, that array (and the buffer's offset and limit) are used; otherwise,
 +   * a new backing array is created and a relative bulk get is performed to
 +   * transfer the buffer's contents (starting at its current position and
 +   * not beyond its limit).
 +   *
 +   * @param buffer byte buffer
 +   */
    public ArrayByteSequence(ByteBuffer buffer) {
      this.length = buffer.remaining();
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/Column.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Column.java
index 59bb5d0,a29e412..f259912
--- a/core/src/main/java/org/apache/accumulo/core/data/Column.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Column.java
@@@ -16,14 -16,14 +16,13 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
  
  import java.io.DataInput;
  import java.io.DataOutput;
  import java.io.IOException;
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
--
  import org.apache.accumulo.core.data.thrift.TColumn;
  import org.apache.hadoop.io.WritableComparable;
  import org.apache.hadoop.io.WritableComparator;
@@@ -204,23 -158,11 +203,23 @@@ public class Column implements Writable
      return columnVisibility;
    }
    
 +  /**
 +   * Gets a string representation of this column. The family, qualifier, and
 +   * visibility are interpreted as strings using the platform default encoding;
 +   * nulls are interpreted as empty strings.
 +   *
 +   * @return string form of column
 +   */
    public String toString() {
-     return new String(columnFamily == null ? new byte[0] : columnFamily, StandardCharsets.UTF_8) + ":" + new String(columnQualifier == null ? new byte[0] : columnQualifier, StandardCharsets.UTF_8) + ":"
-         + new String(columnVisibility == null ? new byte[0] : columnVisibility, StandardCharsets.UTF_8);
+     return new String(columnFamily == null ? new byte[0] : columnFamily, UTF_8) + ":" + new String(columnQualifier == null ? new byte[0] : columnQualifier, UTF_8) + ":"
+         + new String(columnVisibility == null ? new byte[0] : columnVisibility, UTF_8);
    }
    
 +  /**
 +   * Converts this column to Thrift.
 +   *
 +   * @return Thrift column
 +   */
    public TColumn toThrift() {
      return new TColumn(columnFamily == null ? null : ByteBuffer.wrap(columnFamily), columnQualifier == null ? null : ByteBuffer.wrap(columnQualifier),
          columnVisibility == null ? null : ByteBuffer.wrap(columnVisibility));

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index 71088c2,e6e31eb..e4b089d
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -16,9 -16,8 +16,9 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
- import java.nio.charset.StandardCharsets;
  import java.util.Arrays;
  import java.util.HashSet;
  
@@@ -42,20 -42,10 +42,20 @@@ public class Condition 
    private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
    
  
 +  /**
 +   * Creates a new condition. The initial column value and timestamp are null,
 +   * and the initial column visibility is empty. Characters in the column family
 +   * and column qualifier are encoded as bytes in the condition using UTF-8.
 +   *
 +   * @param cf column family
 +   * @param cq column qualifier
 +   * @throws IllegalArgumentException if any argument is null
 +   */
    public Condition(CharSequence cf, CharSequence cq) {
 -    ArgumentChecker.notNull(cf, cq);
 +    checkArgument(cf != null, "cf is null");
 +    checkArgument(cq != null, "cq is null");
-     this.cf = new ArrayByteSequence(cf.toString().getBytes(StandardCharsets.UTF_8));
-     this.cq = new ArrayByteSequence(cq.toString().getBytes(StandardCharsets.UTF_8));
+     this.cf = new ArrayByteSequence(cf.toString().getBytes(UTF_8));
+     this.cq = new ArrayByteSequence(cq.toString().getBytes(UTF_8));
      this.cv = EMPTY;
    }
    
@@@ -146,17 -94,14 +146,17 @@@
    }
  
    /**
 -   * see {@link #setValue(byte[])}
 +   * This method sets the expected value of a column. In order for the condition to pass the column must exist and have this value. If a value is not set, then
 +   * the column must be absent for the condition to pass. The passed-in character sequence is encoded as UTF-8.
 +   * See {@link #setValue(byte[])}.
     * 
 -   * @return returns this
 +   * @param value value
 +   * @return this condition
 +   * @throws IllegalArgumentException if value is null
     */
 -
    public Condition setValue(CharSequence value) {
 -    ArgumentChecker.notNull(value);
 +    checkArgument(value != null, "value is null");
-     this.val = new ArrayByteSequence(value.toString().getBytes(StandardCharsets.UTF_8));
+     this.val = new ArrayByteSequence(value.toString().getBytes(UTF_8));
      return this;
    }
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Value.java
index 7679dd3,d256107..48eae02
--- a/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@@ -17,15 -17,17 +17,15 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
  
  import java.io.DataInput;
  import java.io.DataOutput;
  import java.io.IOException;
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.List;
  
 -import com.google.common.base.Preconditions;
 -
  import org.apache.hadoop.io.BytesWritable;
  import org.apache.hadoop.io.WritableComparable;
  import org.apache.hadoop.io.WritableComparator;
@@@ -225,12 -202,12 +225,12 @@@ public class Value implements WritableC
      }
      return false;
    }
 -  
 +
    @Override
    public String toString() {
-     return new String(get(), StandardCharsets.UTF_8);
+     return new String(get(), UTF_8);
    }
 -  
 +
    /**
     * A Comparator optimized for Value.
     */

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
index 1bc54f2,654689c..90506c3
--- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
@@@ -18,6 -18,8 +18,8 @@@
  
  package org.apache.accumulo.core.file;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.DataInputStream;
  import java.io.DataOutputStream;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
index 4062c3a,f19a97c..b3607ef
--- a/core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Collection;
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
index 3c948c6,bc96e57..3c6c6d2
--- a/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.iterators;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
  import java.io.ByteArrayOutputStream;
  import java.io.DataInputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
index ad43736,cecd87e..01ed523
--- a/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.iterators.conf;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collection;
  import java.util.HashSet;
  import java.util.Set;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
index ccf7d86,1f73e0d..a9cd774
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.math.BigDecimal;
- import java.nio.charset.StandardCharsets;
  import java.util.Iterator;
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
index 2760914,1046480..e859363
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Arrays;
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
index 37157ed,d541454..732a76c
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.Map;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
index 47b3c77,47f3cfd..4db379b
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
index 9aff04f,5b43838..8d56889
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.io.UnsupportedEncodingException;
- import java.nio.charset.StandardCharsets;
  import java.util.Map;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
index ee2e1ae,3f317ea..3c24f0e
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Collection;
  import java.util.HashSet;
  import java.util.Map;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
index bcd085e,0b6c183..9bdc883
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
  import java.io.ByteArrayOutputStream;
  import java.io.DataInputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 17e3c30,bab900e..6b51ac5
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
index 4a10511,91f9416..878aa8e
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Map;
  
  import org.apache.accumulo.core.client.IteratorSetting;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
index 6330dca,0000000..491fda5
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
@@@ -1,298 -1,0 +1,298 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.replication;
 +
- import java.nio.charset.CharacterCodingException;
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
 +
++import java.nio.charset.CharacterCodingException;
 +import org.apache.accumulo.core.client.ScannerBase;
 +import org.apache.accumulo.core.client.lexicoder.ULongLexicoder;
 +import org.apache.accumulo.core.data.ArrayByteSequence;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.base.Preconditions;
 +
 +/**
 + * 
 + */
 +public class ReplicationSchema {
 +  private static final Logger log = LoggerFactory.getLogger(ReplicationSchema.class);
 +
 +  /**
 +   * Portion of a file that must be replication to the given target: peer and some identifying location on that peer, e.g. remote table ID
 +   * <p>
 +   * <code>hdfs://localhost:8020/accumulo/wal/tserver+port/WAL work:serialized_ReplicationTarget [] -> Status Protobuf</code>
 +   */
 +  public static class WorkSection {
 +    public static final Text NAME = new Text("work");
 +    private static final ByteSequence BYTE_SEQ_NAME = new ArrayByteSequence("work");
 +
 +    public static void getFile(Key k, Text buff) {
 +      Preconditions.checkNotNull(k);
 +      Preconditions.checkNotNull(buff);
 +      Preconditions.checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication work key with incorrect colfam");
 +      _getFile(k, buff);
 +    }
 +
 +    public static ReplicationTarget getTarget(Key k) {
 +      return getTarget(k, new Text());
 +    }
 +
 +    public static ReplicationTarget getTarget(Key k, Text buff) {
 +      Preconditions.checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication work key with incorrect colfam");
 +      k.getColumnQualifier(buff);
 +
 +      return ReplicationTarget.from(buff);
 +    }
 +
 +    /**
 +     * Limit the scanner to only pull replication work records
 +     */
 +    public static void limit(ScannerBase scanner) {
 +      scanner.fetchColumnFamily(NAME);
 +    }
 +
 +    public static Mutation add(Mutation m, Text serializedTarget, Value v) {
 +      m.put(NAME, serializedTarget, v);
 +      return m;
 +    }
 +  }
 +
 +  /**
 +   * Holds replication markers tracking status for files
 +   * <p>
 +   * <code>hdfs://localhost:8020/accumulo/wal/tserver+port/WAL repl:local_table_id [] -> Status Protobuf</code>
 +   */
 +  public static class StatusSection {
 +    public static final Text NAME = new Text("repl");
 +    private static final ByteSequence BYTE_SEQ_NAME = new ArrayByteSequence("repl");
 +
 +    /**
 +     * Extract the table ID from the key (inefficiently if called repeatedly)
 +     * 
 +     * @param k
 +     *          Key to extract from
 +     * @return The table ID
 +     * @see #getTableId(Key,Text)
 +     */
 +    public static String getTableId(Key k) {
 +      Text buff = new Text();
 +      getTableId(k, buff);
 +      return buff.toString();
 +    }
 +
 +    /**
 +     * Extract the table ID from the key into the given {@link Text}
 +     * 
 +     * @param k
 +     *          Key to extract from
 +     * @param buff
 +     *          Text to place table ID into
 +     */
 +    public static void getTableId(Key k, Text buff) {
 +      Preconditions.checkNotNull(k);
 +      Preconditions.checkNotNull(buff);
 +
 +      k.getColumnQualifier(buff);
 +    }
 +
 +    /**
 +     * Extract the file name from the row suffix into the given {@link Text}
 +     * 
 +     * @param k
 +     *          Key to extract from
 +     * @param buff
 +     *          Text to place file name into
 +     */
 +    public static void getFile(Key k, Text buff) {
 +      Preconditions.checkNotNull(k);
 +      Preconditions.checkNotNull(buff);
 +      Preconditions.checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication status key with incorrect colfam");
 +
 +      _getFile(k, buff);
 +    }
 +
 +    /**
 +     * Limit the scanner to only return Status records
 +     */
 +    public static void limit(ScannerBase scanner) {
 +      scanner.fetchColumnFamily(NAME);
 +    }
 +
 +    public static Mutation add(Mutation m, Text tableId, Value v) {
 +      m.put(NAME, tableId, v);
 +      return m;
 +    }
 +  }
 +
 +  /**
 +   * Holds the order in which files needed for replication were closed. The intent is to be able to guarantee that files which were closed earlier were
 +   * replicated first and we don't replay data in the wrong order on our peers
 +   * <p>
 +   * <code>encodedTimeOfClosure\x00hdfs://localhost:8020/accumulo/wal/tserver+port/WAL order:source_table_id [] -> Status Protobuf</code>
 +   */
 +  public static class OrderSection {
 +    public static final Text NAME = new Text("order");
 +    public static final Text ROW_SEPARATOR = new Text(new byte[]{0});
 +    private static final ULongLexicoder longEncoder = new ULongLexicoder();
 +
 +    /**
 +     * Extract the table ID from the given key (inefficiently if called repeatedly)
 +     * 
 +     * @param k
 +     *          OrderSection Key
 +     * @return source table id
 +     */
 +    public static String getTableId(Key k) {
 +      Text buff = new Text();
 +      getTableId(k, buff);
 +      return buff.toString();
 +    }
 +
 +    /**
 +     * Extract the table ID from the given key
 +     * 
 +     * @param k
 +     *          OrderSection key
 +     * @param buff
 +     *          Text to place table ID into
 +     */
 +    public static void getTableId(Key k, Text buff) {
 +      Preconditions.checkNotNull(k);
 +      Preconditions.checkNotNull(buff);
 +
 +      k.getColumnQualifier(buff);
 +    }
 +
 +    /**
 +     * Limit the scanner to only return Order records
 +     */
 +    public static void limit(ScannerBase scanner) {
 +      scanner.fetchColumnFamily(NAME);
 +    }
 +
 +    /**
 +     * Creates the Mutation for the Order section for the given file and time
 +     * 
 +     * @param file
 +     *          Filename
 +     * @param timeInMillis
 +     *          Time in millis that the file was closed
 +     * @return Mutation for the Order section
 +     */
 +    public static Mutation createMutation(String file, long timeInMillis) {
 +      Preconditions.checkNotNull(file);
 +      Preconditions.checkArgument(timeInMillis >= 0, "timeInMillis must be greater than zero");
 +
 +      // Encode the time so it sorts properly
 +      byte[] rowPrefix = longEncoder.encode(timeInMillis);
 +      Text row = new Text(rowPrefix);
 +
 +      // Normalize the file using Path
 +      Path p = new Path(file);
 +      String pathString = p.toUri().toString();
 +
 +      log.trace("Normalized {} into {}", file, pathString);
 +
 +      // Append the file as a suffix to the row
-       row.append((ROW_SEPARATOR + pathString).getBytes(StandardCharsets.UTF_8), 0, pathString.length() + ROW_SEPARATOR.getLength());
++      row.append((ROW_SEPARATOR + pathString).getBytes(UTF_8), 0, pathString.length() + ROW_SEPARATOR.getLength());
 +
 +      // Make the mutation and add the column update
 +      return new Mutation(row);
 +    }
 +
 +    /**
 +     * Add a column update to the given mutation with the provided tableId and value
 +     * 
 +     * @param m
 +     *          Mutation for OrderSection
 +     * @param tableId
 +     *          Source table id
 +     * @param v
 +     *          Serialized Status msg
 +     * @return The original Mutation
 +     */
 +    public static Mutation add(Mutation m, Text tableId, Value v) {
 +      m.put(NAME, tableId, v);
 +      return m;
 +    }
 +
 +    public static long getTimeClosed(Key k) {
 +      return getTimeClosed(k, new Text());
 +    }
 +
 +    public static long getTimeClosed(Key k, Text buff) {
 +      k.getRow(buff);
 +      int offset = 0;
 +      // find the last offset
 +      while (true) {
 +        int nextOffset = buff.find(ROW_SEPARATOR.toString(), offset + 1);
 +        if (-1 == nextOffset) {
 +          break;
 +        }
 +        offset = nextOffset;
 +      }
 +
 +      if (-1 == offset) {
 +        throw new IllegalArgumentException("Row does not contain expected separator for OrderSection");
 +      }
 +
 +      byte[] encodedLong = new byte[offset];
 +      System.arraycopy(buff.getBytes(), 0, encodedLong, 0, offset);
 +      return longEncoder.decode(encodedLong);
 +    }
 +
 +    public static String getFile(Key k) {
 +      Text buff = new Text();
 +      return getFile(k, buff);
 +    }
 +
 +    public static String getFile(Key k, Text buff) {
 +      k.getRow(buff);
 +      int offset = 0;
 +      // find the last offset
 +      while (true) {
 +        int nextOffset = buff.find(ROW_SEPARATOR.toString(), offset + 1);
 +        if (-1 == nextOffset) {
 +          break;
 +        }
 +        offset = nextOffset;
 +      }
 +
 +      if (-1 == offset) {
 +        throw new IllegalArgumentException("Row does not contain expected separator for OrderSection");
 +      }
 +
 +      try {
 +        return Text.decode(buff.getBytes(), offset + 1, buff.getLength() - (offset + 1));
 +      } catch (CharacterCodingException e) {
 +        throw new IllegalArgumentException("Could not decode file path", e);
 +      }
 +    }
 +  }
 +
 +  private static void _getFile(Key k, Text buff) {
 +    k.getRow(buff);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
index f882a8d,0000000..9c57ad9
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
@@@ -1,188 -1,0 +1,188 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.replication;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.DataInput;
 +import java.io.DataOutput;
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
- 
 +import org.apache.hadoop.io.DataInputBuffer;
 +import org.apache.hadoop.io.DataOutputBuffer;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.io.WritableUtils;
 +
 +/**
 + * Container for where some work needs to be replicated
 + */
 +public class ReplicationTarget implements Writable {
 +
 +  private String peerName;
 +  private String remoteIdentifier;
 +  private String sourceTableId;
 +
 +  public ReplicationTarget() { }
 +
 +  public ReplicationTarget(String peerName, String remoteIdentifier, String sourceTableId) {
 +    this.peerName = peerName;
 +    this.remoteIdentifier = remoteIdentifier;
 +    this.sourceTableId = sourceTableId;
 +  }
 +
 +  public String getPeerName() {
 +    return peerName;
 +  }
 +
 +  public void setPeerName(String peerName) {
 +    this.peerName = peerName;
 +  }
 +
 +  public String getRemoteIdentifier() {
 +    return remoteIdentifier;
 +  }
 +
 +  public void setRemoteIdentifier(String remoteIdentifier) {
 +    this.remoteIdentifier = remoteIdentifier;
 +  }
 +
 +  public String getSourceTableId() {
 +    return sourceTableId;
 +  }
 +
 +  public void setSourceTableId(String sourceTableId) {
 +    this.sourceTableId = sourceTableId;
 +  }
 +
 +  @Override
 +  public void write(DataOutput out) throws IOException {
 +    if (null == peerName) {
 +      out.writeBoolean(false);
 +    } else {
 +      out.writeBoolean(true);
 +      WritableUtils.writeString(out, peerName);
 +    }
 +
 +    if (null == remoteIdentifier) {
 +      out.writeBoolean(false);
 +    } else {
 +      out.writeBoolean(true);
 +      WritableUtils.writeString(out, remoteIdentifier);
 +    }
 +
 +    if (null == sourceTableId) {
 +      out.writeBoolean(false);
 +    } else {
 +      out.writeBoolean(true);
 +      WritableUtils.writeString(out, sourceTableId);
 +    }
 +  }
 +
 +  @Override
 +  public void readFields(DataInput in) throws IOException {
 +    if (in.readBoolean()) {
 +      this.peerName = WritableUtils.readString(in);
 +    }
 +    if (in.readBoolean()) {
 +      this.remoteIdentifier = WritableUtils.readString(in);
 +    }
 +    if (in.readBoolean()) {
 +      this.sourceTableId = WritableUtils.readString(in);
 +    }
 +  }
 +
 +  @Override
 +  public String toString() {
 +    StringBuilder sb = new StringBuilder(64);
 +    sb.append("Remote Name: ").append(peerName).append(" Remote identifier: ").append(remoteIdentifier).append(" Source Table ID: ").append(sourceTableId);
 +    return sb.toString();
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    return peerName.hashCode() ^ remoteIdentifier.hashCode() ^ sourceTableId.hashCode();
 +  }
 +
 +  @Override
 +  public boolean equals(Object o) {
 +    if (o instanceof ReplicationTarget) {
 +      ReplicationTarget other = (ReplicationTarget) o;
 +
 +      return peerName.equals(other.peerName) && remoteIdentifier.equals(other.remoteIdentifier) && sourceTableId.equals(other.sourceTableId);
 +    }
 +
 +    return false;
 +  }
 +
 +  /**
 +   * Deserialize a ReplicationTarget
 +   * @param t Serialized copy
 +   * @return the deserialized version
 +   */
 +  public static ReplicationTarget from(Text t) {
 +    ReplicationTarget target = new ReplicationTarget();
 +    DataInputBuffer buffer = new DataInputBuffer();
 +    buffer.reset(t.getBytes(), t.getLength());
 +
 +    try {
 +      target.readFields(buffer);
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +
 +    return target;
 +  }
 +
 +  /**
 +   * Deserialize a ReplicationTarget
 +   * @param s Serialized copy
 +   * @return the deserialized version
 +   */
 +  public static ReplicationTarget from(String s) {
 +    ReplicationTarget target = new ReplicationTarget();
 +    DataInputBuffer buffer = new DataInputBuffer();
-     buffer.reset(s.getBytes(StandardCharsets.UTF_8), s.length());
++    buffer.reset(s.getBytes(UTF_8), s.length());
 +
 +    try {
 +      target.readFields(buffer);
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +
 +    return target;
 +  }
 +
 +  /**
 +   * Convenience method to serialize a ReplicationTarget to {@link Text} using the {@link Writable} methods without caring about
 +   * performance penalties due to excessive object creation
 +   * @return The serialized representation of the object
 +   */
 +  public Text toText() {
 +    DataOutputBuffer buffer = new DataOutputBuffer();
 +
 +    try {
 +      this.write(buffer);
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +
 +    Text t = new Text();
 +    // Throw it in a text for the mutation
 +    t.set(buffer.getData(), 0, buffer.getLength());
 +    return t;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
index 54ddcd9,7073d11..b6ce5b7
--- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
@@@ -16,7 -16,7 +16,8 @@@
   */
  package org.apache.accumulo.core.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import java.io.Serializable;
  import java.nio.ByteBuffer;
@@@ -145,9 -145,9 +145,9 @@@ public class Authorizations implements 
     */
    public Authorizations(byte[] authorizations) {
  
 -    ArgumentChecker.notNull(authorizations);
 +    checkArgument(authorizations != null, "authorizations is null");
  
-     String authsString = new String(authorizations, StandardCharsets.UTF_8);
+     String authsString = new String(authorizations, UTF_8);
      if (authsString.startsWith(HEADER)) {
        // it's the new format
        authsString = authsString.substring(HEADER.length());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index 152e067,e8f4e97..3e930f5
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.Serializable;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Credentials.java
index 7981ccf,0ecc31d..525a958
--- a/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
@@@ -16,9 -16,10 +16,9 @@@
   */
  package org.apache.accumulo.core.security;
  
- import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
+ import java.nio.ByteBuffer;
 -
  import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
index 1d29119,4f91dac..c4fe5ac
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.security;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.HashSet;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
index 1d54ae4,178de91..ca24884
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.security;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.text.ParseException;
  
  /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
index 4c8b837,4c2f5c9..9586eaa
--- a/core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.trace;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.List;
  import java.util.Map;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
index fc3c114,b1047d9..4b7db82
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
index 1a34628,8771832..dbf04dc
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index b70eef0,26a8c73..cfac8fe
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@@ -16,12 -16,13 +16,12 @@@
   */
  package org.apache.accumulo.core.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.PrintStream;
- import java.nio.charset.StandardCharsets;
--
  import jline.console.ConsoleReader;
  
  import org.apache.accumulo.core.cli.ClientOpts.Password;


[10/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index 9537f4e,0000000..83e83cf
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -1,1315 -1,0 +1,1317 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicInteger;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.ThriftTransportPool;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.master.thrift.MasterClientService.Iface;
 +import org.apache.accumulo.core.master.thrift.MasterClientService.Processor;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.master.thrift.MasterState;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.AgeOffStore;
 +import org.apache.accumulo.fate.Fate;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.master.recovery.RecoveryManager;
 +import org.apache.accumulo.master.state.TableCounts;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.init.Initialize;
 +import org.apache.accumulo.server.master.LiveTServerSet;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
 +import org.apache.accumulo.server.master.balancer.TabletBalancer;
 +import org.apache.accumulo.server.master.state.CurrentState;
 +import org.apache.accumulo.server.master.state.DeadServerList;
 +import org.apache.accumulo.server.master.state.MergeInfo;
 +import org.apache.accumulo.server.master.state.MergeState;
 +import org.apache.accumulo.server.master.state.MetaDataStateStore;
 +import org.apache.accumulo.server.master.state.RootTabletStateStore;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.state.TabletLocationState;
 +import org.apache.accumulo.server.master.state.TabletMigration;
 +import org.apache.accumulo.server.master.state.TabletState;
 +import org.apache.accumulo.server.master.state.ZooStore;
 +import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.security.handler.ZKPermHandler;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.accumulo.server.tables.TableObserver;
 +import org.apache.accumulo.server.util.DefaultMap;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.RpcWrapper;
 +import org.apache.accumulo.server.util.TServerUtils;
 +import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.start.classloader.vfs.ContextManager;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.DataInputBuffer;
 +import org.apache.hadoop.io.DataOutputBuffer;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.server.TServer;
 +import org.apache.thrift.transport.TTransportException;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.google.common.collect.Iterables;
 +
 +/**
 + * The Master is responsible for assigning and balancing tablets to tablet servers.
 + *
 + * The master will also coordinate log recoveries and reports general status.
 + */
 +public class Master implements LiveTServerSet.Listener, TableObserver, CurrentState {
 +
 +  final static Logger log = Logger.getLogger(Master.class);
 +
 +  final static int ONE_SECOND = 1000;
 +  final private static Text METADATA_TABLE_ID = new Text(MetadataTable.ID);
 +  final private static Text ROOT_TABLE_ID = new Text(RootTable.ID);
 +  final static long TIME_TO_WAIT_BETWEEN_SCANS = 60 * ONE_SECOND;
 +  final private static long TIME_BETWEEN_MIGRATION_CLEANUPS = 5 * 60 * ONE_SECOND;
 +  final static long WAIT_BETWEEN_ERRORS = ONE_SECOND;
 +  final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;
 +  final private static int MAX_CLEANUP_WAIT_TIME = ONE_SECOND;
 +  final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = ONE_SECOND;
 +  final static int MAX_TSERVER_WORK_CHUNK = 5000;
 +  final private static int MAX_BAD_STATUS_COUNT = 3;
 +
 +  final VolumeManager fs;
 +  final private Instance instance;
 +  final private String hostname;
 +  final LiveTServerSet tserverSet;
 +  final private List<TabletGroupWatcher> watchers = new ArrayList<TabletGroupWatcher>();
 +  final SecurityOperation security;
 +  final Map<TServerInstance,AtomicInteger> badServers = Collections.synchronizedMap(new DefaultMap<TServerInstance,AtomicInteger>(new AtomicInteger()));
 +  final Set<TServerInstance> serversToShutdown = Collections.synchronizedSet(new HashSet<TServerInstance>());
 +  final SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,TServerInstance>());
 +  final EventCoordinator nextEvent = new EventCoordinator();
 +  final private Object mergeLock = new Object();
 +  RecoveryManager recoveryManager = null;
 +
 +  ZooLock masterLock = null;
 +  private TServer clientService = null;
 +  TabletBalancer tabletBalancer;
 +
 +  private MasterState state = MasterState.INITIAL;
 +
 +  Fate<Master> fate;
 +
 +  volatile SortedMap<TServerInstance,TabletServerStatus> tserverStatus = Collections.unmodifiableSortedMap(new TreeMap<TServerInstance,TabletServerStatus>());
 +
 +  synchronized MasterState getMasterState() {
 +    return state;
 +  }
 +
 +  public boolean stillMaster() {
 +    return getMasterState() != MasterState.STOP;
 +  }
 +
 +  static final boolean X = true;
 +  static final boolean _ = false;
 +  // @formatter:off
 +  static final boolean transitionOK[][] = {
 +      //                              INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
 +      /* INITIAL */                   {X,     X,        _,        _,      _,         _,          X},
 +      /* HAVE_LOCK */                 {_,     X,        X,        X,      _,         _,          X},
 +      /* SAFE_MODE */                 {_,     _,        X,        X,      X,         _,          X},
 +      /* NORMAL */                    {_,     _,        X,        X,      X,         _,          X},
 +      /* UNLOAD_METADATA_TABLETS */   {_,     _,        X,        X,      X,         X,          X},
 +      /* UNLOAD_ROOT_TABLET */        {_,     _,        _,        X,      X,         X,          X},
 +      /* STOP */                      {_,     _,        _,        _,      _,         X,          X}};
 +  //@formatter:on
 +  synchronized void setMasterState(MasterState newState) {
 +    if (state.equals(newState))
 +      return;
 +    if (!transitionOK[state.ordinal()][newState.ordinal()]) {
 +      log.error("Programmer error: master should not transition from " + state + " to " + newState);
 +    }
 +    MasterState oldState = state;
 +    state = newState;
 +    nextEvent.event("State changed from %s to %s", oldState, newState);
 +    if (newState == MasterState.STOP) {
 +      // Give the server a little time before shutdown so the client
 +      // thread requesting the stop can return
 +      SimpleTimer.getInstance().schedule(new Runnable() {
 +        @Override
 +        public void run() {
 +          // This frees the main thread and will cause the master to exit
 +          clientService.stop();
 +          Master.this.nextEvent.event("stopped event loop");
 +        }
 +
 +      }, 100l, 1000l);
 +    }
 +
 +    if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {
 +      upgradeZookeeper();
 +    }
 +
 +    if (oldState != newState && (newState == MasterState.NORMAL)) {
 +      upgradeMetadata();
 +    }
 +  }
 +
 +  private void moveRootTabletToRootTable(IZooReaderWriter zoo) throws Exception {
 +    String dirZPath = ZooUtil.getRoot(instance) + RootTable.ZROOT_TABLET_PATH;
 +
 +    if (!zoo.exists(dirZPath)) {
 +      Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
 +      if (fs.exists(oldPath)) {
 +        String newPath = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
 +        fs.mkdirs(new Path(newPath));
 +        if (!fs.rename(oldPath, new Path(newPath))) {
 +          throw new IOException("Failed to move root tablet from " + oldPath + " to " + newPath);
 +        }
 +
 +        log.info("Upgrade renamed " + oldPath + " to " + newPath);
 +      }
 +
 +      Path location = null;
 +
 +      for (String basePath : ServerConstants.getTablesDirs()) {
 +        Path path = new Path(basePath + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION);
 +        if (fs.exists(path)) {
 +          if (location != null) {
 +            throw new IllegalStateException("Root table at multiple locations " + location + " " + path);
 +          }
 +
 +          location = path;
 +        }
 +      }
 +
 +      if (location == null)
 +        throw new IllegalStateException("Failed to find root tablet");
 +
 +      log.info("Upgrade setting root table location in zookeeper " + location);
 +      zoo.putPersistentData(dirZPath, location.toString().getBytes(), NodeExistsPolicy.FAIL);
 +    }
 +  }
 +
 +  private boolean haveUpgradedZooKeeper = false;
 +
 +  private void upgradeZookeeper() {
 +    // 1.5.1 and 1.6.0 both do some state checking after obtaining the zoolock for the
 +    // monitor and before starting up. It's not tied to the data version at all (and would
 +    // introduce unnecessary complexity to try to make the master do it), but be aware
 +    // that the master is not the only thing that may alter zookeeper before starting.
 +
 +    final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
 +    if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
 +      // This Master hasn't started Fate yet, so any outstanding transactions must be from before the upgrade.
 +      // Change to Guava's Verify once we use Guava 17.
 +      if (null != fate) {
 +        throw new IllegalStateException(
 +            "Access to Fate should not have been initialized prior to the Master transitioning to active. Please save all logs and file a bug.");
 +      }
 +      Accumulo.abortIfFateTransactions();
 +      try {
 +        log.info("Upgrading zookeeper");
 +
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +        final String zooRoot = ZooUtil.getRoot(instance);
 +
 +        if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
 +          log.debug("Handling updates for version " + ServerConstants.TWO_DATA_VERSIONS_AGO);
 +
 +          log.debug("Cleaning out remnants of logger role.");
 +          zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
 +          zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
 +
 +          final byte[] zero = new byte[] {'0'};
 +          log.debug("Initializing recovery area.");
 +          zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
 +
 +          for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
 +            log.debug("Prepping table " + id + " for compaction cancellations.");
 +            zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
 +          }
 +        }
 +
 +        // create initial namespaces
 +        String namespaces = ZooUtil.getRoot(instance) + Constants.ZNAMESPACES;
 +        zoo.putPersistentData(namespaces, new byte[0], NodeExistsPolicy.SKIP);
 +        for (Pair<String,String> namespace : Iterables.concat(
 +            Collections.singleton(new Pair<String,String>(Namespaces.ACCUMULO_NAMESPACE, Namespaces.ACCUMULO_NAMESPACE_ID)),
 +            Collections.singleton(new Pair<String,String>(Namespaces.DEFAULT_NAMESPACE, Namespaces.DEFAULT_NAMESPACE_ID)))) {
 +          String ns = namespace.getFirst();
 +          String id = namespace.getSecond();
 +          log.debug("Upgrade creating namespace \"" + ns + "\" (ID: " + id + ")");
 +          if (!Namespaces.exists(instance, id))
 +            TableManager.prepareNewNamespaceState(instance.getInstanceID(), id, ns, NodeExistsPolicy.SKIP);
 +        }
 +
 +        // create root table
 +        log.debug("Upgrade creating table " + RootTable.NAME + " (ID: " + RootTable.ID + ")");
 +        TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE,
 +            NodeExistsPolicy.SKIP);
 +        Initialize.initMetadataConfig();
 +        // ensure root user can flush root table
 +        security.grantTablePermission(SystemCredentials.get().toThrift(instance), security.getRootUsername(), RootTable.ID, TablePermission.ALTER_TABLE,
 +            Namespaces.ACCUMULO_NAMESPACE_ID);
 +
 +        // put existing tables in the correct namespaces
 +        String tables = ZooUtil.getRoot(instance) + Constants.ZTABLES;
 +        for (String tableId : zoo.getChildren(tables)) {
 +          String targetNamespace = (MetadataTable.ID.equals(tableId) || RootTable.ID.equals(tableId)) ? Namespaces.ACCUMULO_NAMESPACE_ID
 +              : Namespaces.DEFAULT_NAMESPACE_ID;
-           log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), Constants.UTF8) + " (ID: "
++          log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), UTF_8) + " (ID: "
 +              + tableId + ") into namespace with ID " + targetNamespace);
-           zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++          zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(UTF_8), NodeExistsPolicy.SKIP);
 +        }
 +
 +        // rename metadata table
 +        log.debug("Upgrade renaming table " + MetadataTable.OLD_NAME + " (ID: " + MetadataTable.ID + ") to " + MetadataTable.NAME);
-         zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(Constants.UTF8),
++        zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8),
 +            NodeExistsPolicy.OVERWRITE);
 +
 +        moveRootTabletToRootTable(zoo);
 +
 +        // add system namespace permissions to existing users
 +        ZKPermHandler perm = new ZKPermHandler();
 +        perm.initialize(instance.getInstanceID(), true);
 +        String users = ZooUtil.getRoot(instance) + "/users";
 +        for (String user : zoo.getChildren(users)) {
 +          zoo.putPersistentData(users + "/" + user + "/Namespaces", new byte[0], NodeExistsPolicy.SKIP);
 +          perm.grantNamespacePermission(user, Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.READ);
 +        }
 +        perm.grantNamespacePermission("root", Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.ALTER_TABLE);
 +        haveUpgradedZooKeeper = true;
 +      } catch (Exception ex) {
 +        log.fatal("Error performing upgrade", ex);
 +        System.exit(1);
 +      }
 +    }
 +  }
 +
 +  private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
 +  private final CountDownLatch waitForMetadataUpgrade = new CountDownLatch(1);
 +
 +  private final ServerConfiguration serverConfig;
 +
 +  private void upgradeMetadata() {
 +    // we make sure we're only doing the rest of this method once so that we can signal to other threads that an upgrade wasn't needed.
 +    if (upgradeMetadataRunning.compareAndSet(false, true)) {
 +      final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
 +      if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
 +        // sanity check that we passed the Fate verification prior to ZooKeeper upgrade, and that Fate still hasn't been started.
 +        // Change both to use Guava's Verify once we use Guava 17.
 +        if (!haveUpgradedZooKeeper) {
 +          throw new IllegalStateException(
 +              "We should only attempt to upgrade Accumulo's metadata table if we've already upgraded ZooKeeper. Please save all logs and file a bug.");
 +        }
 +        if (null != fate) {
 +          throw new IllegalStateException(
 +              "Access to Fate should not have been initialized prior to the Master finishing upgrades. Please save all logs and file a bug.");
 +        }
 +        Runnable upgradeTask = new Runnable() {
 +          @Override
 +          public void run() {
 +            try {
 +              log.info("Starting to upgrade metadata table.");
 +              if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
 +                log.info("Updating Delete Markers in metadata table for version 1.4");
 +                MetadataTableUtil.moveMetaDeleteMarkersFrom14(instance, SystemCredentials.get());
 +              } else {
 +                log.info("Updating Delete Markers in metadata table.");
 +                MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
 +              }
 +              log.info("Updating persistent data version.");
 +              Accumulo.updateAccumuloVersion(fs, accumuloPersistentVersion);
 +              log.info("Upgrade complete");
 +              waitForMetadataUpgrade.countDown();
 +            } catch (Exception ex) {
 +              log.fatal("Error performing upgrade", ex);
 +              System.exit(1);
 +            }
 +
 +          }
 +        };
 +
 +        // need to run this in a separate thread because a lock is held that prevents metadata tablets from being assigned and this task writes to the
 +        // metadata table
 +        new Thread(upgradeTask).start();
 +      } else {
 +        waitForMetadataUpgrade.countDown();
 +      }
 +    }
 +  }
 +
 +  private int assignedOrHosted(Text tableId) {
 +    int result = 0;
 +    for (TabletGroupWatcher watcher : watchers) {
 +      TableCounts count = watcher.getStats(tableId);
 +      result += count.hosted() + count.assigned();
 +    }
 +    return result;
 +  }
 +
 +  private int totalAssignedOrHosted() {
 +    int result = 0;
 +    for (TabletGroupWatcher watcher : watchers) {
 +      for (TableCounts counts : watcher.getStats().values()) {
 +        result += counts.assigned() + counts.hosted();
 +      }
 +    }
 +    return result;
 +  }
 +
 +  private int nonMetaDataTabletsAssignedOrHosted() {
 +    return totalAssignedOrHosted() - assignedOrHosted(new Text(MetadataTable.ID)) - assignedOrHosted(new Text(RootTable.ID));
 +  }
 +
 +  private int notHosted() {
 +    int result = 0;
 +    for (TabletGroupWatcher watcher : watchers) {
 +      for (TableCounts counts : watcher.getStats().values()) {
 +        result += counts.assigned() + counts.assignedToDeadServers();
 +      }
 +    }
 +    return result;
 +  }
 +
 +  // The number of unassigned tablets that should be assigned: displayed on the monitor page
 +  int displayUnassigned() {
 +    int result = 0;
 +    switch (getMasterState()) {
 +      case NORMAL:
 +        // Count offline tablets for online tables
 +        for (TabletGroupWatcher watcher : watchers) {
 +          TableManager manager = TableManager.getInstance();
 +          for (Entry<Text,TableCounts> entry : watcher.getStats().entrySet()) {
 +            Text tableId = entry.getKey();
 +            TableCounts counts = entry.getValue();
 +            TableState tableState = manager.getTableState(tableId.toString());
 +            if (tableState != null && tableState.equals(TableState.ONLINE)) {
 +              result += counts.unassigned() + counts.assignedToDeadServers() + counts.assigned();
 +            }
 +          }
 +        }
 +        break;
 +      case SAFE_MODE:
 +        // Count offline tablets for the metadata table
 +        for (TabletGroupWatcher watcher : watchers) {
 +          result += watcher.getStats(METADATA_TABLE_ID).unassigned();
 +        }
 +        break;
 +      case UNLOAD_METADATA_TABLETS:
 +      case UNLOAD_ROOT_TABLET:
 +        for (TabletGroupWatcher watcher : watchers) {
 +          result += watcher.getStats(METADATA_TABLE_ID).unassigned();
 +        }
 +        break;
 +      default:
 +        break;
 +    }
 +    return result;
 +  }
 +
 +  public void mustBeOnline(final String tableId) throws ThriftTableOperationException {
 +    Tables.clearCache(instance);
 +    if (!Tables.getTableState(instance, tableId).equals(TableState.ONLINE))
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.MERGE, TableOperationExceptionType.OFFLINE, "table is not online");
 +  }
 +
 +  public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
 +    return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
 +  }
 +
 +  private Master(ServerConfiguration config, VolumeManager fs, String hostname) throws IOException {
 +    this.serverConfig = config;
 +    this.instance = config.getInstance();
 +    this.fs = fs;
 +    this.hostname = hostname;
 +
 +    AccumuloConfiguration aconf = serverConfig.getConfiguration();
 +
 +    log.info("Version " + Constants.VERSION);
 +    log.info("Instance " + instance.getInstanceID());
 +    ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
 +    security = AuditedSecurityOperation.getInstance();
 +    tserverSet = new LiveTServerSet(instance, config.getConfiguration(), this);
 +    this.tabletBalancer = aconf.instantiateClassProperty(Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
 +    this.tabletBalancer.init(serverConfig);
 +
 +    try {
 +      AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() {
 +        @Override
 +        public Iterator<Entry<String,String>> iterator() {
 +          return getSystemConfiguration().iterator();
 +        }
 +      }));
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +
 +  }
 +
 +  public TServerConnection getConnection(TServerInstance server) {
 +    return tserverSet.getConnection(server);
 +  }
 +
 +  public MergeInfo getMergeInfo(Text tableId) {
 +    synchronized (mergeLock) {
 +      try {
 +        String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
 +        if (!ZooReaderWriter.getInstance().exists(path))
 +          return new MergeInfo();
 +        byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat());
 +        DataInputBuffer in = new DataInputBuffer();
 +        in.reset(data, data.length);
 +        MergeInfo info = new MergeInfo();
 +        info.readFields(in);
 +        return info;
 +      } catch (KeeperException.NoNodeException ex) {
 +        log.info("Error reading merge state, it probably just finished");
 +        return new MergeInfo();
 +      } catch (Exception ex) {
 +        log.warn("Unexpected error reading merge state", ex);
 +        return new MergeInfo();
 +      }
 +    }
 +  }
 +
 +  public void setMergeState(MergeInfo info, MergeState state) throws IOException, KeeperException, InterruptedException {
 +    synchronized (mergeLock) {
 +      String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + info.getExtent().getTableId().toString() + "/merge";
 +      info.setState(state);
 +      if (state.equals(MergeState.NONE)) {
 +        ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
 +      } else {
 +        DataOutputBuffer out = new DataOutputBuffer();
 +        try {
 +          info.write(out);
 +        } catch (IOException ex) {
 +          throw new RuntimeException("Unlikely", ex);
 +        }
 +        ZooReaderWriter.getInstance().putPersistentData(path, out.getData(),
 +            state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE);
 +      }
 +      mergeLock.notifyAll();
 +    }
 +    nextEvent.event("Merge state of %s set to %s", info.getExtent(), state);
 +  }
 +
 +  public void clearMergeState(Text tableId) throws IOException, KeeperException, InterruptedException {
 +    synchronized (mergeLock) {
 +      String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
 +      ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
 +      mergeLock.notifyAll();
 +    }
 +    nextEvent.event("Merge state of %s cleared", tableId);
 +  }
 +
 +  void setMasterGoalState(MasterGoalState state) {
 +    try {
 +      ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, state.name().getBytes(),
 +          NodeExistsPolicy.OVERWRITE);
 +    } catch (Exception ex) {
 +      log.error("Unable to set master goal state in zookeeper");
 +    }
 +  }
 +
 +  MasterGoalState getMasterGoalState() {
 +    while (true)
 +      try {
 +        byte[] data = ZooReaderWriter.getInstance().getData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, null);
 +        return MasterGoalState.valueOf(new String(data));
 +      } catch (Exception e) {
 +        log.error("Problem getting real goal state: " + e);
 +        UtilWaitThread.sleep(1000);
 +      }
 +  }
 +
 +  public boolean hasCycled(long time) {
 +    for (TabletGroupWatcher watcher : watchers) {
 +      if (watcher.stats.lastScanFinished() < time)
 +        return false;
 +    }
 +
 +    return true;
 +  }
 +
 +  public void clearMigrations(String tableId) {
 +    synchronized (migrations) {
 +      Iterator<KeyExtent> iterator = migrations.keySet().iterator();
 +      while (iterator.hasNext()) {
 +        KeyExtent extent = iterator.next();
 +        if (extent.getTableId().toString().equals(tableId)) {
 +          iterator.remove();
 +        }
 +      }
 +    }
 +  }
 +
 +  static enum TabletGoalState {
 +    HOSTED, UNASSIGNED, DELETED
 +  };
 +
 +  TabletGoalState getSystemGoalState(TabletLocationState tls) {
 +    switch (getMasterState()) {
 +      case NORMAL:
 +        return TabletGoalState.HOSTED;
 +      case HAVE_LOCK: // fall-through intended
 +      case INITIAL: // fall-through intended
 +      case SAFE_MODE:
 +        if (tls.extent.isMeta())
 +          return TabletGoalState.HOSTED;
 +        return TabletGoalState.UNASSIGNED;
 +      case UNLOAD_METADATA_TABLETS:
 +        if (tls.extent.isRootTablet())
 +          return TabletGoalState.HOSTED;
 +        return TabletGoalState.UNASSIGNED;
 +      case UNLOAD_ROOT_TABLET:
 +        return TabletGoalState.UNASSIGNED;
 +      case STOP:
 +        return TabletGoalState.UNASSIGNED;
 +      default:
 +        throw new IllegalStateException("Unknown Master State");
 +    }
 +  }
 +
 +  TabletGoalState getTableGoalState(KeyExtent extent) {
 +    TableState tableState = TableManager.getInstance().getTableState(extent.getTableId().toString());
 +    if (tableState == null)
 +      return TabletGoalState.DELETED;
 +    switch (tableState) {
 +      case DELETING:
 +        return TabletGoalState.DELETED;
 +      case OFFLINE:
 +      case NEW:
 +        return TabletGoalState.UNASSIGNED;
 +      default:
 +        return TabletGoalState.HOSTED;
 +    }
 +  }
 +
 +  TabletGoalState getGoalState(TabletLocationState tls, MergeInfo mergeInfo) {
 +    KeyExtent extent = tls.extent;
 +    // Shutting down?
 +    TabletGoalState state = getSystemGoalState(tls);
 +    if (state == TabletGoalState.HOSTED) {
 +      if (tls.current != null && serversToShutdown.contains(tls.current)) {
 +        return TabletGoalState.UNASSIGNED;
 +      }
 +      // Handle merge transitions
 +      if (mergeInfo.getExtent() != null) {
 +        log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent));
 +        if (mergeInfo.overlaps(extent)) {
 +          switch (mergeInfo.getState()) {
 +            case NONE:
 +            case COMPLETE:
 +              break;
 +            case STARTED:
 +            case SPLITTING:
 +              return TabletGoalState.HOSTED;
 +            case WAITING_FOR_CHOPPED:
 +              if (tls.getState(onlineTabletServers()).equals(TabletState.HOSTED)) {
 +                if (tls.chopped)
 +                  return TabletGoalState.UNASSIGNED;
 +              } else {
 +                if (tls.chopped && tls.walogs.isEmpty())
 +                  return TabletGoalState.UNASSIGNED;
 +              }
 +
 +              return TabletGoalState.HOSTED;
 +            case WAITING_FOR_OFFLINE:
 +            case MERGING:
 +              return TabletGoalState.UNASSIGNED;
 +          }
 +        }
 +      }
 +
 +      // taking table offline?
 +      state = getTableGoalState(extent);
 +      if (state == TabletGoalState.HOSTED) {
 +        // Maybe this tablet needs to be migrated
 +        TServerInstance dest = migrations.get(extent);
 +        if (dest != null && tls.current != null && !dest.equals(tls.current)) {
 +          return TabletGoalState.UNASSIGNED;
 +        }
 +      }
 +    }
 +    return state;
 +  }
 +
 +  private class MigrationCleanupThread extends Daemon {
 +
 +    @Override
 +    public void run() {
 +      setName("Migration Cleanup Thread");
 +      while (stillMaster()) {
 +        if (!migrations.isEmpty()) {
 +          try {
 +            cleanupOfflineMigrations();
 +            cleanupNonexistentMigrations(getConnector());
 +          } catch (Exception ex) {
 +            log.error("Error cleaning up migrations", ex);
 +          }
 +        }
 +        UtilWaitThread.sleep(TIME_BETWEEN_MIGRATION_CLEANUPS);
 +      }
 +    }
 +
 +    /**
 +     * If a migrating tablet splits, and the tablet dies before sending the master a message, the migration will refer to a non-existing tablet, so it can never
 +     * complete. Periodically scan the metadata table and remove any migrating tablets that no longer exist.
 +     */
 +    private void cleanupNonexistentMigrations(final Connector connector) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +      Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +      TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      Set<KeyExtent> found = new HashSet<KeyExtent>();
 +      for (Entry<Key,Value> entry : scanner) {
 +        KeyExtent extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
 +        if (migrations.containsKey(extent)) {
 +          found.add(extent);
 +        }
 +      }
 +      migrations.keySet().retainAll(found);
 +    }
 +
 +    /**
 +     * If migrating a tablet for a table that is offline, the migration can never succeed because no tablet server will load the tablet. check for offline
 +     * tables and remove their migrations.
 +     */
 +    private void cleanupOfflineMigrations() {
 +      TableManager manager = TableManager.getInstance();
 +      for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
 +        TableState state = manager.getTableState(tableId);
 +        if (TableState.OFFLINE == state) {
 +          clearMigrations(tableId);
 +        }
 +      }
 +    }
 +  }
 +
 +  private class StatusThread extends Daemon {
 +
 +    @Override
 +    public void run() {
 +      setName("Status Thread");
 +      EventCoordinator.Listener eventListener = nextEvent.getListener();
 +      while (stillMaster()) {
 +        long wait = DEFAULT_WAIT_FOR_WATCHER;
 +        try {
 +          switch (getMasterGoalState()) {
 +            case NORMAL:
 +              setMasterState(MasterState.NORMAL);
 +              break;
 +            case SAFE_MODE:
 +              if (getMasterState() == MasterState.NORMAL) {
 +                setMasterState(MasterState.SAFE_MODE);
 +              }
 +              if (getMasterState() == MasterState.HAVE_LOCK) {
 +                setMasterState(MasterState.SAFE_MODE);
 +              }
 +              break;
 +            case CLEAN_STOP:
 +              switch (getMasterState()) {
 +                case NORMAL:
 +                  setMasterState(MasterState.SAFE_MODE);
 +                  break;
 +                case SAFE_MODE: {
 +                  int count = nonMetaDataTabletsAssignedOrHosted();
 +                  log.debug(String.format("There are %d non-metadata tablets assigned or hosted", count));
 +                  if (count == 0)
 +                    setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
 +                }
 +                  break;
 +                case UNLOAD_METADATA_TABLETS: {
 +                  int count = assignedOrHosted(METADATA_TABLE_ID);
 +                  log.debug(String.format("There are %d metadata tablets assigned or hosted", count));
 +                  if (count == 0)
 +                    setMasterState(MasterState.UNLOAD_ROOT_TABLET);
 +                }
 +                  break;
 +                case UNLOAD_ROOT_TABLET: {
 +                  int count = assignedOrHosted(METADATA_TABLE_ID);
 +                  if (count > 0) {
 +                    log.debug(String.format("%d metadata tablets online", count));
 +                    setMasterState(MasterState.UNLOAD_ROOT_TABLET);
 +                  }
 +                  int root_count = assignedOrHosted(ROOT_TABLE_ID);
 +                  if (root_count > 0)
 +                    log.debug("The root tablet is still assigned or hosted");
 +                  if (count + root_count == 0) {
 +                    Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
 +                    log.debug("stopping " + currentServers.size() + " tablet servers");
 +                    for (TServerInstance server : currentServers) {
 +                      try {
 +                        serversToShutdown.add(server);
 +                        tserverSet.getConnection(server).fastHalt(masterLock);
 +                      } catch (TException e) {
 +                        // its probably down, and we don't care
 +                      } finally {
 +                        tserverSet.remove(server);
 +                      }
 +                    }
 +                    if (currentServers.size() == 0)
 +                      setMasterState(MasterState.STOP);
 +                  }
 +                }
 +                  break;
 +                default:
 +                  break;
 +              }
 +          }
 +          wait = updateStatus();
 +          eventListener.waitForEvents(wait);
 +        } catch (Throwable t) {
 +          log.error("Error balancing tablets", t);
 +          UtilWaitThread.sleep(WAIT_BETWEEN_ERRORS);
 +        }
 +      }
 +    }
 +
 +    private long updateStatus() throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +      tserverStatus = Collections.synchronizedSortedMap(gatherTableInformation());
 +      checkForHeldServer(tserverStatus);
 +
 +      if (!badServers.isEmpty()) {
 +        log.debug("not balancing because the balance information is out-of-date " + badServers.keySet());
 +      } else if (notHosted() > 0) {
 +        log.debug("not balancing because there are unhosted tablets: " + notHosted());
 +      } else if (getMasterGoalState() == MasterGoalState.CLEAN_STOP) {
 +        log.debug("not balancing because the master is attempting to stop cleanly");
 +      } else if (!serversToShutdown.isEmpty()) {
 +        log.debug("not balancing while shutting down servers " + serversToShutdown);
 +      } else {
 +        return balanceTablets();
 +      }
 +      return DEFAULT_WAIT_FOR_WATCHER;
 +    }
 +
 +    private void checkForHeldServer(SortedMap<TServerInstance,TabletServerStatus> tserverStatus) {
 +      TServerInstance instance = null;
 +      int crazyHoldTime = 0;
 +      int someHoldTime = 0;
 +      final long maxWait = getSystemConfiguration().getTimeInMillis(Property.TSERV_HOLD_TIME_SUICIDE);
 +      for (Entry<TServerInstance,TabletServerStatus> entry : tserverStatus.entrySet()) {
 +        if (entry.getValue().getHoldTime() > 0) {
 +          someHoldTime++;
 +          if (entry.getValue().getHoldTime() > maxWait) {
 +            instance = entry.getKey();
 +            crazyHoldTime++;
 +          }
 +        }
 +      }
 +      if (crazyHoldTime == 1 && someHoldTime == 1 && tserverStatus.size() > 1) {
 +        log.warn("Tablet server " + instance + " exceeded maximum hold time: attempting to kill it");
 +        try {
 +          TServerConnection connection = tserverSet.getConnection(instance);
 +          if (connection != null)
 +            connection.fastHalt(masterLock);
 +        } catch (TException e) {
 +          log.error(e, e);
 +        }
 +        tserverSet.remove(instance);
 +      }
 +    }
 +
 +    private long balanceTablets() {
 +      List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
 +      Set<KeyExtent> migrationsCopy = new HashSet<KeyExtent>();
 +      synchronized (migrations) {
 +        migrationsCopy.addAll(migrations.keySet());
 +      }
 +      long wait = tabletBalancer.balance(Collections.unmodifiableSortedMap(tserverStatus), Collections.unmodifiableSet(migrationsCopy), migrationsOut);
 +
 +      for (TabletMigration m : TabletBalancer.checkMigrationSanity(tserverStatus.keySet(), migrationsOut)) {
 +        if (migrations.containsKey(m.tablet)) {
 +          log.warn("balancer requested migration more than once, skipping " + m);
 +          continue;
 +        }
 +        migrations.put(m.tablet, m.newServer);
 +        log.debug("migration " + m);
 +      }
 +      if (migrationsOut.size() > 0) {
 +        nextEvent.event("Migrating %d more tablets, %d total", migrationsOut.size(), migrations.size());
 +      }
 +      return wait;
 +    }
 +
 +  }
 +
 +  private SortedMap<TServerInstance,TabletServerStatus> gatherTableInformation() {
 +    long start = System.currentTimeMillis();
 +    SortedMap<TServerInstance,TabletServerStatus> result = new TreeMap<TServerInstance,TabletServerStatus>();
 +    Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
 +    for (TServerInstance server : currentServers) {
 +      try {
 +        Thread t = Thread.currentThread();
 +        String oldName = t.getName();
 +        try {
 +          t.setName("Getting status from " + server);
 +          TServerConnection connection = tserverSet.getConnection(server);
 +          if (connection == null)
 +            throw new IOException("No connection to " + server);
 +          TabletServerStatus status = connection.getTableMap(false);
 +          result.put(server, status);
 +        } finally {
 +          t.setName(oldName);
 +        }
 +      } catch (Exception ex) {
 +        log.error("unable to get tablet server status " + server + " " + ex.toString());
 +        log.debug("unable to get tablet server status " + server, ex);
 +        if (badServers.get(server).incrementAndGet() > MAX_BAD_STATUS_COUNT) {
 +          log.warn("attempting to stop " + server);
 +          try {
 +            TServerConnection connection = tserverSet.getConnection(server);
 +            if (connection != null)
 +              connection.halt(masterLock);
 +          } catch (TTransportException e) {
 +            // ignore: it's probably down
 +          } catch (Exception e) {
 +            log.info("error talking to troublesome tablet server ", e);
 +          }
 +          badServers.remove(server);
 +          tserverSet.remove(server);
 +        }
 +      }
 +    }
 +    synchronized (badServers) {
 +      badServers.keySet().retainAll(currentServers);
 +      badServers.keySet().removeAll(result.keySet());
 +    }
 +    log.debug(String.format("Finished gathering information from %d servers in %.2f seconds", result.size(), (System.currentTimeMillis() - start) / 1000.));
 +    return result;
 +  }
 +
 +  public void run() throws IOException, InterruptedException, KeeperException {
 +    final String zroot = ZooUtil.getRoot(instance);
 +
 +    getMasterLock(zroot + Constants.ZMASTER_LOCK);
 +
 +    recoveryManager = new RecoveryManager(this);
 +
 +    TableManager.getInstance().addObserver(this);
 +
 +    StatusThread statusThread = new StatusThread();
 +    statusThread.start();
 +
 +    MigrationCleanupThread migrationCleanupThread = new MigrationCleanupThread();
 +    migrationCleanupThread.start();
 +
 +    tserverSet.startListeningForTabletServerChanges();
 +
 +    ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, new Watcher() {
 +      @Override
 +      public void process(WatchedEvent event) {
 +        nextEvent.event("Noticed recovery changes", event.getType());
 +        try {
 +          // watcher only fires once, add it back
 +          ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, this);
 +        } catch (Exception e) {
 +          log.error("Failed to add log recovery watcher back", e);
 +        }
 +      }
 +    });
 +
 +    Credentials systemCreds = SystemCredentials.get();
 +    watchers.add(new TabletGroupWatcher(this, new MetaDataStateStore(instance, systemCreds, this), null));
 +    watchers.add(new TabletGroupWatcher(this, new RootTabletStateStore(instance, systemCreds, this), watchers.get(0)));
 +    watchers.add(new TabletGroupWatcher(this, new ZooTabletStateStore(new ZooStore(zroot)), watchers.get(1)));
 +    for (TabletGroupWatcher watcher : watchers) {
 +      watcher.start();
 +    }
 +
 +    // Once we are sure the upgrade is complete, we can safely allow fate use.
 +    waitForMetadataUpgrade.await();
 +
 +    try {
 +      final AgeOffStore<Master> store = new AgeOffStore<Master>(new org.apache.accumulo.fate.ZooStore<Master>(ZooUtil.getRoot(instance) + Constants.ZFATE,
 +          ZooReaderWriter.getInstance()), 1000 * 60 * 60 * 8);
 +
 +      int threads = this.getConfiguration().getConfiguration().getCount(Property.MASTER_FATE_THREADPOOL_SIZE);
 +
 +      fate = new Fate<Master>(this, store, threads);
 +
 +      SimpleTimer.getInstance().schedule(new Runnable() {
 +
 +        @Override
 +        public void run() {
 +          store.ageOff();
 +        }
 +      }, 63000, 63000);
 +    } catch (KeeperException e) {
 +      throw new IOException(e);
 +    } catch (InterruptedException e) {
 +      throw new IOException(e);
 +    }
 +
 +    Processor<Iface> processor = new Processor<Iface>(RpcWrapper.service(new MasterClientServiceHandler(this)));
 +    ServerAddress sa = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_CLIENTPORT, processor, "Master",
 +        "Master Client Service Handler", null, Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
 +    clientService = sa.server;
 +    String address = sa.address.toString();
 +    log.info("Setting master lock data to " + address);
 +    masterLock.replaceLockData(address.getBytes());
 +
 +    while (!clientService.isServing()) {
 +      UtilWaitThread.sleep(100);
 +    }
 +    while (clientService.isServing()) {
 +      UtilWaitThread.sleep(500);
 +    }
 +    log.info("Shutting down fate.");
 +    fate.shutdown();
 +
 +    final long deadline = System.currentTimeMillis() + MAX_CLEANUP_WAIT_TIME;
 +    statusThread.join(remaining(deadline));
 +
 +    // quit, even if the tablet servers somehow jam up and the watchers
 +    // don't stop
 +    for (TabletGroupWatcher watcher : watchers) {
 +      watcher.join(remaining(deadline));
 +    }
 +    log.info("exiting");
 +  }
 +
 +  private long remaining(long deadline) {
 +    return Math.max(1, deadline - System.currentTimeMillis());
 +  }
 +
 +  public ZooLock getMasterLock() {
 +    return masterLock;
 +  }
 +
 +  private static class MasterLockWatcher implements ZooLock.AsyncLockWatcher {
 +
 +    boolean acquiredLock = false;
 +    boolean failedToAcquireLock = false;
 +
 +    @Override
 +    public void lostLock(LockLossReason reason) {
 +      Halt.halt("Master lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
 +    }
 +
 +    @Override
 +    public void unableToMonitorLockNode(final Throwable e) {
 +      Halt.halt(-1, new Runnable() {
 +        @Override
 +        public void run() {
 +          log.fatal("No longer able to monitor master lock node", e);
 +        }
 +      });
 +
 +    }
 +
 +    @Override
 +    public synchronized void acquiredLock() {
 +      log.debug("Acquired master lock");
 +
 +      if (acquiredLock || failedToAcquireLock) {
 +        Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      acquiredLock = true;
 +      notifyAll();
 +    }
 +
 +    @Override
 +    public synchronized void failedToAcquireLock(Exception e) {
 +      log.warn("Failed to get master lock " + e);
 +
 +      if (acquiredLock) {
 +        Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      failedToAcquireLock = true;
 +      notifyAll();
 +    }
 +
 +    public synchronized void waitForChange() {
 +      while (!acquiredLock && !failedToAcquireLock) {
 +        try {
 +          wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +  }
 +
 +  private void getMasterLock(final String zMasterLoc) throws KeeperException, InterruptedException {
 +    log.info("trying to get master lock");
 +
 +    final String masterClientAddress = hostname + ":" + getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
 +
 +    while (true) {
 +
 +      MasterLockWatcher masterLockWatcher = new MasterLockWatcher();
 +      masterLock = new ZooLock(zMasterLoc);
 +      masterLock.lockAsync(masterLockWatcher, masterClientAddress.getBytes());
 +
 +      masterLockWatcher.waitForChange();
 +
 +      if (masterLockWatcher.acquiredLock) {
 +        break;
 +      }
 +
 +      if (!masterLockWatcher.failedToAcquireLock) {
 +        throw new IllegalStateException("master lock in unknown state");
 +      }
 +
 +      masterLock.tryToCancelAsyncLockOrUnlock();
 +
 +      UtilWaitThread.sleep(TIME_TO_WAIT_BETWEEN_LOCK_CHECKS);
 +    }
 +
 +    setMasterState(MasterState.HAVE_LOCK);
 +  }
 +
 +  public static void main(String[] args) throws Exception {
 +    try {
 +      SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +
 +      ServerOpts opts = new ServerOpts();
 +      final String app = "master";
 +      opts.parseArgs(app, args);
 +      String hostname = opts.getAddress();
 +      Accumulo.setupLogging(app);
 +      Instance instance = HdfsZooInstance.getInstance();
 +      ServerConfiguration conf = new ServerConfiguration(instance);
 +      VolumeManager fs = VolumeManagerImpl.get();
 +      Accumulo.init(fs, conf, app);
 +      Master master = new Master(conf, fs, hostname);
 +      Accumulo.enableTracing(hostname, app);
 +      master.run();
 +    } catch (Exception ex) {
 +      log.error("Unexpected exception, exiting", ex);
 +      System.exit(1);
 +    }
 +  }
 +
 +  @Override
 +  public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
 +    DeadServerList obit = new DeadServerList(ZooUtil.getRoot(instance) + Constants.ZDEADTSERVERS);
 +    if (added.size() > 0) {
 +      log.info("New servers: " + added);
 +      for (TServerInstance up : added)
 +        obit.delete(up.hostPort());
 +    }
 +    for (TServerInstance dead : deleted) {
 +      String cause = "unexpected failure";
 +      if (serversToShutdown.contains(dead))
 +        cause = "clean shutdown"; // maybe an incorrect assumption
 +      if (!getMasterGoalState().equals(MasterGoalState.CLEAN_STOP))
 +        obit.post(dead.hostPort(), cause);
 +    }
 +
 +    Set<TServerInstance> unexpected = new HashSet<TServerInstance>(deleted);
 +    unexpected.removeAll(this.serversToShutdown);
 +    if (unexpected.size() > 0) {
 +      if (stillMaster() && !getMasterGoalState().equals(MasterGoalState.CLEAN_STOP)) {
 +        log.warn("Lost servers " + unexpected);
 +      }
 +    }
 +    serversToShutdown.removeAll(deleted);
 +    badServers.keySet().removeAll(deleted);
 +    // clear out any bad server with the same host/port as a new server
 +    synchronized (badServers) {
 +      cleanListByHostAndPort(badServers.keySet(), deleted, added);
 +    }
 +    synchronized (serversToShutdown) {
 +      cleanListByHostAndPort(serversToShutdown, deleted, added);
 +    }
 +
 +    synchronized (migrations) {
 +      Iterator<Entry<KeyExtent,TServerInstance>> iter = migrations.entrySet().iterator();
 +      while (iter.hasNext()) {
 +        Entry<KeyExtent,TServerInstance> entry = iter.next();
 +        if (deleted.contains(entry.getValue())) {
 +          log.info("Canceling migration of " + entry.getKey() + " to " + entry.getValue());
 +          iter.remove();
 +        }
 +      }
 +    }
 +    nextEvent.event("There are now %d tablet servers", current.size());
 +  }
 +
 +  private static void cleanListByHostAndPort(Collection<TServerInstance> badServers, Set<TServerInstance> deleted, Set<TServerInstance> added) {
 +    Iterator<TServerInstance> badIter = badServers.iterator();
 +    while (badIter.hasNext()) {
 +      TServerInstance bad = badIter.next();
 +      for (TServerInstance add : added) {
 +        if (bad.hostPort().equals(add.hostPort())) {
 +          badIter.remove();
 +          break;
 +        }
 +      }
 +      for (TServerInstance del : deleted) {
 +        if (bad.hostPort().equals(del.hostPort())) {
 +          badIter.remove();
 +          break;
 +        }
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void stateChanged(String tableId, TableState state) {
 +    nextEvent.event("Table state in zookeeper changed for %s to %s", tableId, state);
 +    if (TableState.OFFLINE == state) {
 +      clearMigrations(tableId);
 +    }
 +  }
 +
 +  @Override
 +  public void initialize(Map<String,TableState> tableIdToStateMap) {}
 +
 +  @Override
 +  public void sessionExpired() {}
 +
 +  @Override
 +  public Set<String> onlineTables() {
 +    Set<String> result = new HashSet<String>();
 +    if (getMasterState() != MasterState.NORMAL) {
 +      if (getMasterState() != MasterState.UNLOAD_METADATA_TABLETS)
 +        result.add(MetadataTable.ID);
 +      if (getMasterState() != MasterState.UNLOAD_ROOT_TABLET)
 +        result.add(RootTable.ID);
 +      return result;
 +    }
 +    TableManager manager = TableManager.getInstance();
 +
 +    for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
 +      TableState state = manager.getTableState(tableId);
 +      if (state != null) {
 +        if (state == TableState.ONLINE)
 +          result.add(tableId);
 +      }
 +    }
 +    return result;
 +  }
 +
 +  @Override
 +  public Set<TServerInstance> onlineTabletServers() {
 +    return tserverSet.getCurrentServers();
 +  }
 +
 +  @Override
 +  public Collection<MergeInfo> merges() {
 +    List<MergeInfo> result = new ArrayList<MergeInfo>();
 +    for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
 +      result.add(getMergeInfo(new Text(tableId)));
 +    }
 +    return result;
 +  }
 +
 +  // recovers state from the persistent transaction to shutdown a server
 +  public void shutdownTServer(TServerInstance server) {
 +    nextEvent.event("Tablet Server shutdown requested for %s", server);
 +    serversToShutdown.add(server);
 +  }
 +
 +  public EventCoordinator getEventCoordinator() {
 +    return nextEvent;
 +  }
 +
 +  public Instance getInstance() {
 +    return this.instance;
 +  }
 +
 +  public AccumuloConfiguration getSystemConfiguration() {
 +    return serverConfig.getConfiguration();
 +  }
 +
 +  public ServerConfiguration getConfiguration() {
 +    return serverConfig;
 +  }
 +
 +  public VolumeManager getFileSystem() {
 +    return this.fs;
 +  }
 +
 +  public void assignedTablet(KeyExtent extent) {
 +    if (extent.isMeta()) {
 +      if (getMasterState().equals(MasterState.UNLOAD_ROOT_TABLET)) {
 +        setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
 +      }
 +    }
 +    if (extent.isRootTablet()) {
 +      // probably too late, but try anyhow
 +      if (getMasterState().equals(MasterState.STOP)) {
 +        setMasterState(MasterState.UNLOAD_ROOT_TABLET);
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index 0c2e1f0,0000000..5e478cb
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@@ -1,190 -1,0 +1,192 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.recovery;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.ScheduledExecutorService;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.util.NamingThreadFactory;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.log.SortedLogState;
 +import org.apache.accumulo.server.master.recovery.HadoopLogCloser;
 +import org.apache.accumulo.server.master.recovery.LogCloser;
 +import org.apache.accumulo.server.master.recovery.RecoveryPath;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class RecoveryManager {
 +
 +  private static Logger log = Logger.getLogger(RecoveryManager.class);
 +
 +  private Map<String,Long> recoveryDelay = new HashMap<String,Long>();
 +  private Set<String> closeTasksQueued = new HashSet<String>();
 +  private Set<String> sortsQueued = new HashSet<String>();
 +  private ScheduledExecutorService executor;
 +  private Master master;
 +  private ZooCache zooCache;
 +
 +  public RecoveryManager(Master master) {
 +    this.master = master;
 +    executor = Executors.newScheduledThreadPool(4, new NamingThreadFactory("Walog sort starter "));
 +    zooCache = new ZooCache();
 +    try {
 +      List<String> workIDs = new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).getWorkQueued();
 +      sortsQueued.addAll(workIDs);
 +    } catch (Exception e) {
 +      log.warn(e, e);
 +    }
 +  }
 +
 +  private class LogSortTask implements Runnable {
 +    private String source;
 +    private String destination;
 +    private String sortId;
 +    private LogCloser closer;
 +
 +    public LogSortTask(LogCloser closer, String source, String destination, String sortId) {
 +      this.closer = closer;
 +      this.source = source;
 +      this.destination = destination;
 +      this.sortId = sortId;
 +    }
 +
 +    @Override
 +    public void run() {
 +      boolean rescheduled = false;
 +      try {
 +
 +        long time = closer.close(master.getConfiguration().getConfiguration(), master.getFileSystem(), new Path(source));
 +
 +        if (time > 0) {
 +          executor.schedule(this, time, TimeUnit.MILLISECONDS);
 +          rescheduled = true;
 +        } else {
 +          initiateSort(sortId, source, destination);
 +        }
 +      } catch (FileNotFoundException e) {
 +        log.debug("Unable to initate log sort for " + source + ": " + e);
 +      } catch (Exception e) {
 +        log.warn("Failed to initiate log sort " + source, e);
 +      } finally {
 +        if (!rescheduled) {
 +          synchronized (RecoveryManager.this) {
 +            closeTasksQueued.remove(sortId);
 +          }
 +        }
 +      }
 +    }
 +
 +  }
 +
 +  private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
 +    String work = source + "|" + destination;
-     new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(Constants.UTF8));
++    new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(UTF_8));
 +
 +    synchronized (this) {
 +      sortsQueued.add(sortId);
 +    }
 +
 +    final String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId;
 +    log.info("Created zookeeper entry " + path + " with data " + work);
 +  }
 +
 +  public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>> walogs) throws IOException {
 +    boolean recoveryNeeded = false;
 +
 +    for (Collection<String> logs : walogs) {
 +      for (String walog : logs) {
 +
 +        String switchedWalog = VolumeUtil.switchVolume(walog, FileType.WAL, ServerConstants.getVolumeReplacements());
 +        if (switchedWalog != null) {
 +          // replaces the volume used for sorting, but do not change entry in metadata table. When the tablet loads it will change the metadata table entry. If
 +          // the tablet has the same replacement config, then it will find the sorted log.
 +          log.info("Volume replaced " + walog + " -> " + switchedWalog);
 +          walog = switchedWalog;
 +        }
 +
 +        String parts[] = walog.split("/");
 +        String sortId = parts[parts.length - 1];
 +        String filename = master.getFileSystem().getFullPath(FileType.WAL, walog).toString();
 +        String dest = RecoveryPath.getRecoveryPath(master.getFileSystem(), new Path(filename)).toString();
 +        log.debug("Recovering " + filename + " to " + dest);
 +
 +        boolean sortQueued;
 +        synchronized (this) {
 +          sortQueued = sortsQueued.contains(sortId);
 +        }
 +
 +        if (sortQueued && zooCache.get(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId) == null) {
 +          synchronized (this) {
 +            sortsQueued.remove(sortId);
 +          }
 +        }
 +
 +        if (master.getFileSystem().exists(SortedLogState.getFinishedMarkerPath(dest))) {
 +          synchronized (this) {
 +            closeTasksQueued.remove(sortId);
 +            recoveryDelay.remove(sortId);
 +            sortsQueued.remove(sortId);
 +          }
 +          continue;
 +        }
 +
 +        recoveryNeeded = true;
 +        synchronized (this) {
 +          if (!closeTasksQueued.contains(sortId) && !sortsQueued.contains(sortId)) {
 +            AccumuloConfiguration aconf = master.getConfiguration().getConfiguration();
 +            LogCloser closer = aconf.instantiateClassProperty(Property.MASTER_WALOG_CLOSER_IMPLEMETATION, LogCloser.class, new HadoopLogCloser());
 +            Long delay = recoveryDelay.get(sortId);
 +            if (delay == null) {
 +              delay = master.getSystemConfiguration().getTimeInMillis(Property.MASTER_RECOVERY_DELAY);
 +            } else {
 +              delay = Math.min(2 * delay, 1000 * 60 * 5l);
 +            }
 +
 +            log.info("Starting recovery of " + filename + " (in : " + (delay / 1000) + "s), tablet " + extent + " holds a reference");
 +
 +            executor.schedule(new LogSortTask(closer, filename, dest, sortId), delay, TimeUnit.MILLISECONDS);
 +            closeTasksQueued.add(sortId);
 +            recoveryDelay.put(sortId, delay);
 +          }
 +        }
 +      }
 +    }
 +    return recoveryNeeded;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
index f981bae,0000000..a674557
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
@@@ -1,49 -1,0 +1,51 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.state;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +public class SetGoalState {
 +  
 +  /**
 +   * Utility program that will change the goal state for the master from the command line.
 +   */
 +  public static void main(String[] args) throws Exception {
 +    if (args.length != 1 || MasterGoalState.valueOf(args[0]) == null) {
 +      System.err.println("Usage: accumulo " + SetGoalState.class.getName() + " [NORMAL|SAFE_MODE|CLEAN_STOP]");
 +      System.exit(-1);
 +    }
 +    SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Accumulo.waitForZookeeperAndHdfs(fs);
-     ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(Constants.UTF8),
++    ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(UTF_8),
 +        NodeExistsPolicy.OVERWRITE);
 +  }
 +  
 +}


[19/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index cf1f0ea,0000000..da83386
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,407 -1,0 +1,409 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.net.URI;
 +import java.net.URISyntaxException;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.util.StringUtils;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum ConnectorInfo {
 +    IS_CONFIGURED, PRINCIPAL, TOKEN,
 +  }
 +
 +  public static enum TokenSource {
 +    FILE, INLINE;
 +
 +    private String prefix;
 +
 +    private TokenSource() {
 +      prefix = name().toLowerCase() + ":";
 +    }
 +
 +    public String prefix() {
 +      return prefix;
 +    }
 +  }
 +
 +  /**
 +   * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum InstanceOpts {
 +    TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
 +  }
 +
 +  /**
 +   * Configuration keys for general configuration options.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum GeneralOpts {
 +    LOG_LEVEL,
 +    VISIBILITY_CACHE_SIZE
 +  }
 +
 +  /**
 +   * Provides a configuration key for a given feature enum, prefixed by the implementingClass
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param e
 +   *          the enum used to provide the unique part of the configuration key
 +   * @return the configuration key
 +   * @since 1.6.0
 +   */
 +  protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
 +    return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
 +  }
 +
 +  /**
 +  * Provides a configuration key for a given feature enum.
 +  * 
 +  * @param e
 +  *          the enum used to provide the unique part of the configuration key
 +  * @return the configuration key
 +  */
 +  protected static String enumToConfKey(Enum<?> e) {
 +	  return  e.getDeclaringClass().getSimpleName() + "." +  StringUtils.camelize(e.name().toLowerCase());
 +  }
 +
 +  /**
 +   * Sets the connector information needed to communicate with Accumulo in this job.
 +   * 
 +   * <p>
 +   * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
 +   * conversion to a string, and is not intended to be secure.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param principal
 +   *          a valid Accumulo user name
 +   * @param token
 +   *          the user's password
 +   * @since 1.6.0
 +   */
 +  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
 +      throws AccumuloSecurityException {
 +    if (isConnectorInfoSet(implementingClass, conf))
 +      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
 +
 +    ArgumentChecker.notNull(principal, token);
 +    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
 +        TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
 +  }
 +
 +  /**
 +   * Sets the connector information needed to communicate with Accumulo in this job.
 +   * 
 +   * <p>
 +   * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
 +   * Configuration. Token file created with "bin/accumulo create-token".
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param principal
 +   *          a valid Accumulo user name
 +   * @param tokenFile
 +   *          the path to the token file in DFS
 +   * @since 1.6.0
 +   */
 +  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
 +    if (isConnectorInfoSet(implementingClass, conf))
 +      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
 +
 +    ArgumentChecker.notNull(principal, tokenFile);
 +
 +    try {
 +      DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
 +    } catch (URISyntaxException e) {
 +      throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
 +    }
 +
 +    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
 +  }
 +
 +  /**
 +   * Determines if the connector info has already been set for this instance.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the connector info has already been set, false otherwise
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
 +  }
 +
 +  /**
 +   * Gets the user name from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the principal
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
 +  }
 +
 +  /**
 +   * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the principal's authentication token
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   * @see #setConnectorInfo(Class, Configuration, String, String)
 +   */
 +  public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
 +    String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
 +    if (token == null || token.isEmpty())
 +      return null;
 +    if (token.startsWith(TokenSource.INLINE.prefix())) {
 +      String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
 +      if (args.length == 2)
-         return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(Constants.UTF8)));
++        return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(UTF_8)));
 +    } else if (token.startsWith(TokenSource.FILE.prefix())) {
 +      String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
 +      return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
 +    }
 +
 +    throw new IllegalStateException("Token was not properly serialized into the configuration");
 +  }
 +
 +  /**
 +   * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
 +   * 
 +   * @param conf
 +   *          the Hadoop context for the configured job
 +   * @return path to the token file as a String
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
 +    FSDataInputStream in = null;
 +    try {
 +      URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
 +      Path path = null;
 +      for (URI u : uris) {
 +        if (u.toString().equals(tokenFile)) {
 +          path = new Path(u);
 +        }
 +      }
 +      if (path == null) {
 +        throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
 +      }
 +      FileSystem fs = FileSystem.get(conf);
 +      in = fs.open(path);
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
 +    }
 +    java.util.Scanner fileScanner = new java.util.Scanner(in);
 +    try {
 +      while (fileScanner.hasNextLine()) {
 +        Credentials creds = Credentials.deserialize(fileScanner.nextLine());
 +        if (principal.equals(creds.getPrincipal())) {
 +          return creds.getToken();
 +        }
 +      }
 +      throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
 +    } finally {
 +      if (fileScanner != null && fileScanner.ioException() == null)
 +        fileScanner.close();
 +      else if (fileScanner.ioException() != null)
 +        throw new RuntimeException(fileScanner.ioException());
 +    }
 +  }
 +
 +  /**
 +   * Configures a {@link ZooKeeperInstance} for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param clientConfig
 +   *          client configuration for specifying connection timeouts, SSL connection options, etc.
 +   * @since 1.6.0
 +   */
 +  public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
 +    String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
 +    if (!conf.get(key, "").isEmpty())
 +      throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
 +    conf.set(key, "ZooKeeperInstance");
 +    if (clientConfig != null) {
 +      conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
 +    }
 +  }
 +
 +  /**
 +   * Configures a {@link MockInstance} for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param instanceName
 +   *          the Accumulo instance name
 +   * @since 1.6.0
 +   */
 +  public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
 +    String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
 +    if (!conf.get(key, "").isEmpty())
 +      throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
 +    conf.set(key, "MockInstance");
 +
 +    ArgumentChecker.notNull(instanceName);
 +    conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
 +  }
 +
 +  /**
 +   * Initializes an Accumulo {@link Instance} based on the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return an Accumulo instance
 +   * @since 1.6.0
 +   * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
 +   * @see #setMockInstance(Class, Configuration, String)
 +   */
 +  public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
 +    String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
 +    if ("MockInstance".equals(instanceType))
 +      return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
 +    else if ("ZooKeeperInstance".equals(instanceType)) {
 +      String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
 +      if (clientConfigString == null) {
 +        String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
 +        String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
 +        return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
 +      } else {
 +        return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
 +      }
 +    } else if (instanceType.isEmpty())
 +      throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
 +    else
 +      throw new IllegalStateException("Unrecognized instance type " + instanceType);
 +  }
 +
 +  /**
 +   * Sets the log level for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param level
 +   *          the logging level
 +   * @since 1.6.0
 +   */
 +  public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
 +    ArgumentChecker.notNull(level);
 +    Logger.getLogger(implementingClass).setLevel(level);
 +    conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
 +  }
 +
 +  /**
 +   * Gets the log level from this configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the log level
 +   * @since 1.6.0
 +   * @see #setLogLevel(Class, Configuration, Level)
 +   */
 +  public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
 +    return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
 +  }
 +
 +  /**
 +   * Sets the valid visibility count for this job.
 +   * 
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param visibilityCacheSize
 +   *          the LRU cache size
 +   */
 +  public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
 +    conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
 +  }
 +
 +  /**
 +   * Gets the valid visibility count for this job.
 +   * 
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the valid visibility count
 +   */
 +  public static int getVisibilityCacheSize(Configuration conf) {
 +    return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
index 1ed23e3,0000000..f8a5d03
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@@ -1,795 -1,0 +1,795 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.core.util.ArgumentChecker.notNull;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.StringTokenizer;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientSideIteratorScanner;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.TabletLocator;
 +import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
 +import org.apache.accumulo.core.client.mock.impl.MockTabletLocator;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.MapWritable;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.util.StringUtils;
 +
 +import com.google.common.collect.Maps;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class InputConfigurator extends ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link Scanner}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum ScanOpts {
 +    TABLE_NAME, AUTHORIZATIONS, RANGES, COLUMNS, ITERATORS, TABLE_CONFIGS
 +  }
 +
 +  /**
 +   * Configuration keys for various features.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum Features {
 +    AUTO_ADJUST_RANGES, SCAN_ISOLATION, USE_LOCAL_ITERATORS, SCAN_OFFLINE
 +  }
 +
 +  /**
 +   * Sets the name of the input table, over which this job will scan.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table to use when the tablename is null in the write call
 +   * @since 1.6.0
 +   */
 +  public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
 +    notNull(tableName);
 +    conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
 +  }
 +
 +  /**
 +   * Sets the name of the input table, over which this job will scan.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @since 1.6.0
 +   */
 +  public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
 +  }
 +
 +  /**
 +   * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param auths
 +   *          the user's authorizations
 +   * @since 1.6.0
 +   */
 +  public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf, Authorizations auths) {
 +    if (auths != null && !auths.isEmpty())
 +      conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
 +  }
 +
 +  /**
 +   * Gets the authorizations to set for the scans from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the Accumulo scan authorizations
 +   * @since 1.6.0
 +   * @see #setScanAuthorizations(Class, Configuration, Authorizations)
 +   */
 +  public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
 +    String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
-     return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(Constants.UTF8));
++    return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(UTF_8));
 +  }
 +
 +  /**
 +   * Sets the input ranges to scan on all input tables for this job. If not set, the entire table will be scanned.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param ranges
 +   *          the ranges that will be mapped over
 +   * @throws IllegalArgumentException
 +   *           if the ranges cannot be encoded into base 64
 +   * @since 1.6.0
 +   */
 +  public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
 +    notNull(ranges);
 +
 +    ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
 +    try {
 +      for (Range r : ranges) {
 +        ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +        r.write(new DataOutputStream(baos));
 +        rangeStrings.add(Base64.encodeBase64String(baos.toByteArray()));
 +      }
 +      conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES), rangeStrings.toArray(new String[0]));
 +    } catch (IOException ex) {
 +      throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
 +    }
 +  }
 +
 +  /**
 +   * Gets the ranges to scan over from a job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the ranges
 +   * @throws IOException
 +   *           if the ranges have been encoded improperly
 +   * @since 1.6.0
 +   * @see #setRanges(Class, Configuration, Collection)
 +   */
 +  public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
 +
 +    Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
 +    List<Range> ranges = new ArrayList<Range>();
 +    for (String rangeString : encodedRanges) {
 +      ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes()));
 +      Range range = new Range();
 +      range.readFields(new DataInputStream(bais));
 +      ranges.add(range);
 +    }
 +    return ranges;
 +  }
 +
 +  /**
 +   * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return a list of iterators
 +   * @since 1.6.0
 +   * @see #addIterator(Class, Configuration, IteratorSetting)
 +   */
 +  public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
 +    String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
 +
 +    // If no iterators are present, return an empty list
 +    if (iterators == null || iterators.isEmpty())
 +      return new ArrayList<IteratorSetting>();
 +
 +    // Compose the set of iterators encoded in the job configuration
 +    StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
 +    List<IteratorSetting> list = new ArrayList<IteratorSetting>();
 +    try {
 +      while (tokens.hasMoreTokens()) {
 +        String itstring = tokens.nextToken();
 +        ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
 +        list.add(new IteratorSetting(new DataInputStream(bais)));
 +        bais.close();
 +      }
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("couldn't decode iterator settings");
 +    }
 +    return list;
 +  }
 +
 +  /**
 +   * Restricts the columns that will be mapped over for the single input table on this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param columnFamilyColumnQualifierPairs
 +   *          a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
 +   *          selected. An empty set is the default and is equivalent to scanning the all columns.
 +   * @throws IllegalArgumentException
 +   *           if the column family is null
 +   * @since 1.6.0
 +   */
 +  public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
 +    notNull(columnFamilyColumnQualifierPairs);
 +    String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
 +    conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
 +  }
 +
 +  public static String[] serializeColumns(Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
 +    notNull(columnFamilyColumnQualifierPairs);
 +    ArrayList<String> columnStrings = new ArrayList<String>(columnFamilyColumnQualifierPairs.size());
 +    for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
 +
 +      if (column.getFirst() == null)
 +        throw new IllegalArgumentException("Column family can not be null");
 +
 +      String col = Base64.encodeBase64String(TextUtil.getBytes(column.getFirst()));
 +      if (column.getSecond() != null)
 +        col += ":" + Base64.encodeBase64String(TextUtil.getBytes(column.getSecond()));
 +      columnStrings.add(col);
 +    }
 +
 +    return columnStrings.toArray(new String[0]);
 +  }
 +
 +  /**
 +   * Gets the columns to be mapped over from this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return a set of columns
 +   * @since 1.6.0
 +   * @see #fetchColumns(Class, Configuration, Collection)
 +   */
 +  public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass, Configuration conf) {
 +    notNull(conf);
 +    String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
 +    List<String> serialized = new ArrayList<String>();
 +    if (confValue != null) {
 +      // Split and include any trailing empty strings to allow empty column families
 +      for (String val : confValue.split(",", -1)) {
 +        serialized.add(val);
 +      }
 +    }
 +    return deserializeFetchedColumns(serialized);
 +  }
 +
 +  public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
 +    Set<Pair<Text,Text>> columns = new HashSet<Pair<Text,Text>>();
 +
 +    if (null == serialized) {
 +      return columns;
 +    }
 +
 +    for (String col : serialized) {
 +      int idx = col.indexOf(":");
-       Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8)));
-       Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8)));
++      Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(UTF_8)));
++      Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(UTF_8)));
 +      columns.add(new Pair<Text,Text>(cf, cq));
 +    }
 +    return columns;
 +  }
 +
 +  /**
 +   * Encode an iterator on the input for the single input table associated with this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param cfg
 +   *          the configuration of the iterator
 +   * @throws IllegalArgumentException
 +   *           if the iterator can't be serialized into the configuration
 +   * @since 1.6.0
 +   */
 +  public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    String newIter;
 +    try {
 +      cfg.write(new DataOutputStream(baos));
 +      newIter = Base64.encodeBase64String(baos.toByteArray());
 +      baos.close();
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("unable to serialize IteratorSetting");
 +    }
 +
 +    String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
 +    String iterators = conf.get(confKey);
 +    // No iterators specified yet, create a new string
 +    if (iterators == null || iterators.isEmpty()) {
 +      iterators = newIter;
 +    } else {
 +      // append the next iterator & reset
 +      iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
 +    }
 +    // Store the iterators w/ the job
 +    conf.set(confKey, iterators);
 +  }
 +
 +  /**
 +   * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
 +   * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
 +   * 
 +   * <p>
 +   * By default, this feature is <b>enabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @see #setRanges(Class, Configuration, Collection)
 +   * @since 1.6.0
 +   */
 +  public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has auto-adjust ranges enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return false if the feature is disabled, true otherwise
 +   * @since 1.6.0
 +   * @see #setAutoAdjustRanges(Class, Configuration, boolean)
 +   */
 +  public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
 +  }
 +
 +  /**
 +   * Controls the use of the {@link IsolatedScanner} in this job.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has isolation enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setScanIsolation(Class, Configuration, boolean)
 +   */
 +  public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
 +  }
 +
 +  /**
 +   * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
 +   * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration uses local iterators.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setLocalIterators(Class, Configuration, boolean)
 +   */
 +  public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
 +  }
 +
 +  /**
 +   * <p>
 +   * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
 +   * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
 +   * fail.
 +   * 
 +   * <p>
 +   * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
 +   * 
 +   * <p>
 +   * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
 +   * on the mapper's classpath.
 +   * 
 +   * <p>
 +   * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
 +   * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
 +   * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
 +   * 
 +   * <p>
 +   * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
 +   * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has the offline table scan feature enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setOfflineTableScan(Class, Configuration, boolean)
 +   */
 +  public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
 +  }
 +
 +  /**
 +   * Sets configurations for multiple tables at a time.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param configs
 +   *          an array of {@link InputTableConfig} objects to associate with the job
 +   * @since 1.6.0
 +   */
 +  public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String,InputTableConfig> configs) {
 +    MapWritable mapWritable = new MapWritable();
 +    for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
 +      mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
 +
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    try {
 +      mapWritable.write(new DataOutputStream(baos));
 +    } catch (IOException e) {
 +      throw new IllegalStateException("Table configuration could not be serialized.");
 +    }
 +
 +    String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
 +    conf.set(confKey, Base64.encodeBase64String(baos.toByteArray()));
 +  }
 +
 +  /**
 +   * Returns all {@link InputTableConfig} objects associated with this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return all of the table query configs for the job
 +   * @since 1.6.0
 +   */
 +  public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
 +    Map<String,InputTableConfig> configs = new HashMap<String,InputTableConfig>();
 +    Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
 +    if (defaultConfig != null)
 +      configs.put(defaultConfig.getKey(), defaultConfig.getValue());
 +    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
 +    MapWritable mapWritable = new MapWritable();
 +    if (configString != null) {
 +      try {
 +        byte[] bytes = Base64.decodeBase64(configString.getBytes());
 +        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
 +        mapWritable.readFields(new DataInputStream(bais));
 +        bais.close();
 +      } catch (IOException e) {
 +        throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
 +      }
 +    }
 +    for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
 +      configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());
 +
 +    return configs;
 +  }
 +
 +  /**
 +   * Returns the {@link InputTableConfig} for the given table
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table name for which to fetch the table query config
 +   * @return the table query config for the given table name (if it exists) and null if it does not
 +   * @since 1.6.0
 +   */
 +  public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf, String tableName) {
 +    Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
 +    return queryConfigs.get(tableName);
 +  }
 +
 +  /**
 +   * Initializes an Accumulo {@link TabletLocator} based on the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableId
 +   *          The table id for which to initialize the {@link TabletLocator}
 +   * @return an Accumulo tablet locator
 +   * @throws TableNotFoundException
 +   *           if the table name set on the configuration doesn't exist
 +   * @since 1.6.0
 +   */
 +  public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf, String tableId) throws TableNotFoundException {
 +    String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
 +    if ("MockInstance".equals(instanceType))
 +      return new MockTabletLocator();
 +    Instance instance = getInstance(implementingClass, conf);
 +    return TabletLocator.getLocator(instance, new Text(tableId));
 +  }
 +
 +  // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
 +  /**
 +   * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @throws IOException
 +   *           if the context is improperly configured
 +   * @since 1.6.0
 +   */
 +  public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
 +
 +    Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
 +    if (!isConnectorInfoSet(implementingClass, conf))
 +      throw new IOException("Input info has not been set.");
 +    String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
 +    if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
 +      throw new IOException("Instance info has not been set.");
 +    // validate that we can connect as configured
 +    try {
 +      String principal = getPrincipal(implementingClass, conf);
 +      AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
 +      Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
 +      if (!c.securityOperations().authenticateUser(principal, token))
 +        throw new IOException("Unable to authenticate user");
 +
 +      if (getInputTableConfigs(implementingClass, conf).size() == 0)
 +        throw new IOException("No table set.");
 +
 +      for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
 +        if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
 +          throw new IOException("Unable to access table");
 +      }
 +      for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
 +        InputTableConfig tableConfig = tableConfigEntry.getValue();
 +        if (!tableConfig.shouldUseLocalIterators()) {
 +          if (tableConfig.getIterators() != null) {
 +            for (IteratorSetting iter : tableConfig.getIterators()) {
 +              if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
 +                throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
 +            }
 +          }
 +        }
 +      }
 +    } catch (AccumuloException e) {
 +      throw new IOException(e);
 +    } catch (AccumuloSecurityException e) {
 +      throw new IOException(e);
 +    } catch (TableNotFoundException e) {
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  /**
 +   * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the configuration based on the properties set using the single-table
 +   * input methods.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop instance for which to retrieve the configuration
 +   * @return the config object built from the single input table properties set on the job
 +   * @since 1.6.0
 +   */
 +  protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(Class<?> implementingClass, Configuration conf) {
 +    String tableName = getInputTableName(implementingClass, conf);
 +    if (tableName != null) {
 +      InputTableConfig queryConfig = new InputTableConfig();
 +      List<IteratorSetting> itrs = getIterators(implementingClass, conf);
 +      if (itrs != null)
 +        queryConfig.setIterators(itrs);
 +      Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
 +      if (columns != null)
 +        queryConfig.fetchColumns(columns);
 +      List<Range> ranges = null;
 +      try {
 +        ranges = getRanges(implementingClass, conf);
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      if (ranges != null)
 +        queryConfig.setRanges(ranges);
 +
 +      queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)).setUseIsolatedScanners(isIsolated(implementingClass, conf))
 +          .setUseLocalIterators(usesLocalIterators(implementingClass, conf)).setOfflineScan(isOfflineScan(implementingClass, conf));
 +      return Maps.immutableEntry(tableName, queryConfig);
 +    }
 +    return null;
 +  }
 +
 +  public static Map<String,Map<KeyExtent,List<Range>>> binOffline(String tableId, List<Range> ranges, Instance instance, Connector conn)
 +      throws AccumuloException, TableNotFoundException {
 +    Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
 +
 +    if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
 +      Tables.clearCache(instance);
 +      if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
 +        throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
 +      }
 +    }
 +
 +    for (Range range : ranges) {
 +      Text startRow;
 +
 +      if (range.getStartKey() != null)
 +        startRow = range.getStartKey().getRow();
 +      else
 +        startRow = new Text();
 +
 +      Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
 +      Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +      MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
 +      scanner.setRange(metadataRange);
 +
 +      RowIterator rowIter = new RowIterator(scanner);
 +      KeyExtent lastExtent = null;
 +      while (rowIter.hasNext()) {
 +        Iterator<Map.Entry<Key,Value>> row = rowIter.next();
 +        String last = "";
 +        KeyExtent extent = null;
 +        String location = null;
 +
 +        while (row.hasNext()) {
 +          Map.Entry<Key,Value> entry = row.next();
 +          Key key = entry.getKey();
 +
 +          if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
 +            last = entry.getValue().toString();
 +          }
 +
 +          if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
 +              || key.getColumnFamily().equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
 +            location = entry.getValue().toString();
 +          }
 +
 +          if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
 +            extent = new KeyExtent(key.getRow(), entry.getValue());
 +          }
 +
 +        }
 +
 +        if (location != null)
 +          return null;
 +
 +        if (!extent.getTableId().toString().equals(tableId)) {
 +          throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
 +        }
 +
 +        if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
 +          throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
 +        }
 +
 +        Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
 +        if (tabletRanges == null) {
 +          tabletRanges = new HashMap<KeyExtent,List<Range>>();
 +          binnedRanges.put(last, tabletRanges);
 +        }
 +
 +        List<Range> rangeList = tabletRanges.get(extent);
 +        if (rangeList == null) {
 +          rangeList = new ArrayList<Range>();
 +          tabletRanges.put(extent, rangeList);
 +        }
 +
 +        rangeList.add(range);
 +
 +        if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
 +          break;
 +        }
 +
 +        lastExtent = extent;
 +      }
 +
 +    }
 +    return binnedRanges;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
index 0ba42cd,0000000..26680dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
@@@ -1,204 -1,0 +1,205 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.hadoop.conf.Configuration;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class OutputConfigurator extends ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link BatchWriter}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum WriteOpts {
 +    DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
 +  }
 +
 +  /**
 +   * Configuration keys for various features.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum Features {
 +    CAN_CREATE_TABLES, SIMULATION_MODE
 +  }
 +
 +  /**
 +   * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
 +   * underscores.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table to use when the tablename is null in the write call
 +   * @since 1.6.0
 +   */
 +  public static void setDefaultTableName(Class<?> implementingClass, Configuration conf, String tableName) {
 +    if (tableName != null)
 +      conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
 +  }
 +
 +  /**
 +   * Gets the default table name from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the default table name
 +   * @since 1.6.0
 +   * @see #setDefaultTableName(Class, Configuration, String)
 +   */
 +  public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
 +  }
 +
 +  /**
 +   * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
 +   * used. Setting the configuration multiple times overwrites any previous configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param bwConfig
 +   *          the configuration for the {@link BatchWriter}
 +   * @since 1.6.0
 +   */
 +  public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf, BatchWriterConfig bwConfig) {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    String serialized;
 +    try {
 +      bwConfig.write(new DataOutputStream(baos));
-       serialized = new String(baos.toByteArray(), Constants.UTF8);
++      serialized = new String(baos.toByteArray(), UTF_8);
 +      baos.close();
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
 +    }
 +    conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
 +  }
 +
 +  /**
 +   * Gets the {@link BatchWriterConfig} settings.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the configuration object
 +   * @since 1.6.0
 +   * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
 +   */
 +  public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
 +    String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
 +    BatchWriterConfig bwConfig = new BatchWriterConfig();
 +    if (serialized == null || serialized.isEmpty()) {
 +      return bwConfig;
 +    } else {
 +      try {
-         ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(Constants.UTF8));
++        ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
 +        bwConfig.readFields(new DataInputStream(bais));
 +        bais.close();
 +        return bwConfig;
 +      } catch (IOException e) {
 +        throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setCreateTables(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether tables are permitted to be created as needed.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is disabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setCreateTables(Class, Configuration, boolean)
 +   */
 +  public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
 +  }
 +
 +  /**
 +   * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setSimulationMode(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether this feature is enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setSimulationMode(Class, Configuration, boolean)
 +   */
 +  public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 1541fae,0753282..9e26672
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@@ -26,11 -28,10 +28,10 @@@ import java.util.Arrays
  import java.util.Scanner;
  import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
 -import org.apache.commons.codec.binary.Base64;
 +import org.apache.accumulo.core.client.mapreduce.lib.impl.DistributedCacheHelper;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.hadoop.conf.Configurable;
  import org.apache.hadoop.conf.Configuration;
 -import org.apache.hadoop.filecache.DistributedCache;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.Text;
  import org.apache.hadoop.io.Writable;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
index 0d3aca2,d95a647..95fcf53
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
@@@ -16,20 -16,18 +16,21 @@@
   */
  package org.apache.accumulo.core.client.mock;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
 -import java.io.File;
  import java.io.IOException;
  import java.io.InputStream;
 +import java.io.OutputStream;
  import java.io.Writer;
  
 -import jline.ConsoleReader;
 +import org.apache.commons.io.output.WriterOutputStream;
  
 -import org.apache.accumulo.core.util.shell.Shell;
  import org.apache.commons.cli.CommandLine;
 +import jline.console.ConsoleReader;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.shell.Shell;
 +import org.apache.accumulo.core.util.shell.ShellOptionsJC;
  
  /**
   * An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes.
@@@ -38,29 -36,12 +39,29 @@@ public class MockShell extends Shell 
    private static final String NEWLINE = "\n";
    
    protected InputStream in;
 -  protected Writer writer;
 -  
 -  public MockShell(InputStream in, Writer writer) throws IOException {
 +  protected OutputStream out;
 +
 +  /**
 +   * Will only be set if you use either the Writer constructor or the setWriter(Writer) method
 +   * @deprecated since 1.6.0; use out
 +   */
 +  @Deprecated
 +  protected Writer writer = null;
 +
 +  public MockShell(InputStream in, OutputStream out) throws IOException {
      super();
      this.in = in;
 -    this.writer = writer;
 +    this.out = out;
 +    // we presume they don't use the writer field unless they use the other constructor.
 +  }
 +
 +  /**
 +   * @deprecated since 1.6.0; use OutputStream version
 +   */
 +  @Deprecated
 +  public MockShell(InputStream in, Writer out) throws IOException {
-     this(in, new WriterOutputStream(out, Constants.UTF8.name()));
++    this(in, new WriterOutputStream(out, UTF_8.name()));
 +    this.writer = out;
    }
    
    public boolean config(String... args) {
@@@ -107,7 -79,7 +108,7 @@@
        printInfo();
      
      if (execFile != null) {
-       java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
 -      java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++      java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
        try {
          while (scanner.hasNextLine() && !hasExited()) {
            execCommand(scanner.nextLine(), true, isVerbose());
@@@ -146,20 -118,11 +147,20 @@@
    }
    
    /**
 -   * @param writer
 -   *          the writer to set
 +   * @param out
 +   *          the output stream to set
 +   */
 +  public void setConsoleWriter(OutputStream out) {
 +    this.out = out;
 +  }
 +
 +  /**
 +   * @deprecated since 1.6.0; use the OutputStream version
     */
 -  public void setConsoleWriter(Writer writer) {
 -    this.writer = writer;
 +  @Deprecated
 +  public void setConsoleWriter(Writer out) {
-     setConsoleWriter(new WriterOutputStream(out, Constants.UTF8.name()));
++    setConsoleWriter(new WriterOutputStream(out, UTF_8.name()));
 +    this.writer = out;
    }
    
    /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index bd9feba,05aa0c3..4d73979
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@@ -124,14 -123,10 +125,14 @@@ public class PasswordToken implements A
        throw new RuntimeException(e);
      }
    }
 -  
 -  private void setPassword(CharBuffer charBuffer) {
 +
 +  protected void setPassword(byte[] password) {
 +    this.password = Arrays.copyOf(password, password.length);
 +  }
 +
 +  protected void setPassword(CharBuffer charBuffer) {
      // encode() kicks back a C-string, which is not compatible with the old passwording system
-     ByteBuffer bb = Constants.UTF8.encode(charBuffer);
+     ByteBuffer bb = UTF_8.encode(charBuffer);
      // create array using byter buffer length
      this.password = new byte[bb.remaining()];
      bb.get(this.password);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 41a1f3b,0000000..618ff7e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -1,356 -1,0 +1,357 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.PrintStream;
 +import java.io.UnsupportedEncodingException;
 +import java.util.ArrayList;
 +import java.util.TreeMap;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * This class generates documentation to inform users of the available configuration properties in a presentable form.
 + */
 +class ConfigurationDocGen {
 +  private abstract class Format {
 +
 +    abstract void beginSection(String section);
 +
 +    void endSection() {}
 +
 +    void generate() {
 +      pageHeader();
 +
 +      beginSection("Available Properties");
 +      for (Property prefix : prefixes) {
 +        if (!prefix.isExperimental()) {
 +          prefixSection(prefix);
 +          for (Property prop : sortedProps.values()) {
 +            if (!prop.isExperimental()) {
 +              property(prefix, prop);
 +            }
 +          }
 +        }
 +      }
 +      endSection();
 +
 +      beginSection("Property Types");
 +      propertyTypeDescriptions();
 +      endSection();
 +
 +      pageFooter();
 +      doc.close();
 +    }
 +
 +    abstract String getExt();
 +
 +    void pageFooter() {}
 +
 +    // read static header content from resources and output
 +    void pageHeader() {
 +      appendResource("config-header." + getExt());
 +      doc.println();
 +    }
 +
 +    abstract void prefixSection(Property prefix);
 +
 +    abstract void property(Property prefix, Property prop);
 +
 +    abstract void propertyTypeDescriptions();
 +
 +    abstract String sanitize(String str);
 +
 +  }
 +
 +  private class HTML extends Format {
 +    private boolean highlight;
 +
 +    private void beginRow() {
 +      doc.println(startTag("tr", highlight ? "class='highlight'" : null));
 +      highlight = !highlight;
 +    }
 +
 +    @Override
 +    void beginSection(String section) {
 +      doc.println(t("h1", section, null));
 +      highlight = true;
 +      doc.println("<table>");
 +    }
 +
 +    private String t(String tag, String cell, String options) {
 +      return startTag(tag, options) + cell + "</" + tag + ">";
 +    }
 +
 +    private void cellData(String cell, String options) {
 +      doc.println(t("td", cell, options));
 +    }
 +
 +    private void columnNames(String... names) {
 +      beginRow();
 +      for (String s : names)
 +        doc.println(t("th", s, null));
 +      endRow();
 +    }
 +
 +    private void endRow() {
 +      doc.println("</tr>");
 +    }
 +
 +    @Override
 +    void endSection() {
 +      doc.println("</table>");
 +    }
 +
 +    @Override
 +    String getExt() {
 +      return "html";
 +    }
 +
 +    @Override
 +    void pageFooter() {
 +      doc.println("</body>");
 +      doc.println("</html>");
 +    }
 +
 +    @Override
 +    void pageHeader() {
 +      super.pageHeader();
 +      doc.println("<p>Jump to: ");
 +      String delimiter = "";
 +      for (Property prefix : prefixes) {
 +        if (!prefix.isExperimental()) {
 +          doc.print(delimiter + "<a href='#" + prefix.name() + "'>" + prefix.getKey() + "*</a>");
 +          delimiter = "&nbsp;|&nbsp;";
 +        }
 +      }
 +      doc.println("</p>");
 +    }
 +
 +    @Override
 +    void prefixSection(Property prefix) {
 +      beginRow();
 +      doc.println(t("td", t("span", prefix.getKey() + '*', "id='" + prefix.name() + "' class='large'"), "colspan='5'"
 +          + (prefix.isDeprecated() ? " class='deprecated'" : "")));
 +      endRow();
 +      beginRow();
 +      doc.println(t("td", (prefix.isDeprecated() ? t("b", t("i", "Deprecated. ", null), null) : "") + sanitize(prefix.getDescription()), "colspan='5'"
 +          + (prefix.isDeprecated() ? " class='deprecated'" : "")));
 +      endRow();
 +
 +      switch (prefix) {
 +        case TABLE_CONSTRAINT_PREFIX:
 +          break;
 +        case TABLE_ITERATOR_PREFIX:
 +          break;
 +        case TABLE_LOCALITY_GROUP_PREFIX:
 +          break;
 +        case TABLE_COMPACTION_STRATEGY_PREFIX:
 +          break;
 +        default:
 +          columnNames("Property", "Type", "ZooKeeper Mutable", "Default Value", "Description");
 +          break;
 +      }
 +    }
 +
 +    @Override
 +    void property(Property prefix, Property prop) {
 +      boolean isDeprecated = prefix.isDeprecated() || prop.isDeprecated();
 +      if (prop.getKey().startsWith(prefix.getKey())) {
 +        beginRow();
 +        cellData(prop.getKey(), isDeprecated ? "class='deprecated'" : null);
 +        cellData("<b><a href='#" + prop.getType().name() + "'>" + prop.getType().toString().replaceAll(" ", "&nbsp;") + "</a></b>",
 +            isDeprecated ? "class='deprecated'" : null);
 +        cellData(isZooKeeperMutable(prop), isDeprecated ? "class='deprecated'" : null);
 +        cellData("<pre>" + (prop.getRawDefaultValue().isEmpty() ? "&nbsp;" : sanitize(prop.getRawDefaultValue().replaceAll(" ", "&nbsp;"))) + "</pre>",
 +            isDeprecated ? "class='deprecated'" : null);
 +        cellData((isDeprecated ? "<b><i>Deprecated.</i></b> " : "") + sanitize(prop.getDescription()), isDeprecated ? "class='deprecated'" : null);
 +        endRow();
 +      }
 +
 +    }
 +
 +    @Override
 +    void propertyTypeDescriptions() {
 +      columnNames("Property Type", "Description");
 +      for (PropertyType type : PropertyType.values()) {
 +        if (type == PropertyType.PREFIX)
 +          continue;
 +        beginRow();
 +        cellData("<h3 id='" + type.name() + "'>" + type + "</h3>", null);
 +        cellData(type.getFormatDescription(), null);
 +        endRow();
 +      }
 +    }
 +
 +    @Override
 +    String sanitize(String str) {
 +      return str.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replaceAll("(?:\r\n|\r|\n)", "<br />");
 +    }
 +
 +    private String startTag(String tag, String options) {
 +      return "<" + tag + (options != null ? " " + options : "") + ">";
 +    }
 +
 +  }
 +
 +  private class LaTeX extends Format {
 +    @Override
 +    void beginSection(String section) {
 +      doc.println("\\section{" + section + "}");
 +    }
 +
 +    @Override
 +    String getExt() {
 +      return "tex";
 +    }
 +
 +    @Override
 +    void prefixSection(Property prefix) {
 +      boolean depr = prefix.isDeprecated();
 +      doc.println("\\subsection{" + prefix.getKey() + "*}" + (depr ? " (Deprecated)" : ""));
 +      doc.println((depr ? "\\sout{\\textit{Deprecated.} " : "") + sanitize(prefix.getDescription()) + (depr ? "}" : ""));
 +      doc.println();
 +    }
 +
 +    @Override
 +    void property(Property prefix, Property prop) {
 +      boolean depr = prefix.isDeprecated() || prop.isDeprecated();
 +      if (prop.getKey().startsWith(prefix.getKey())) {
 +        doc.println("\\subsubsection{" + prop.getKey() + "}");
 +        doc.println(strike((depr ? "\\textit{Deprecated.} " : "") + sanitize(prop.getDescription()), depr));
 +        doc.println();
 +        doc.println(strike("\\textit{Type:} " + prop.getType().name() + "\\\\", depr));
 +        doc.println(strike("\\textit{ZooKeeper Mutable:} " + isZooKeeperMutable(prop) + "\\\\", depr));
 +        doc.println(strike("\\textit{Default Value:} " + sanitize(prop.getRawDefaultValue()), depr));
 +        doc.println();
 +      }
 +    }
 +
 +    private String strike(String s, boolean isDeprecated) {
 +      return (isDeprecated ? "\\sout{" : "") + s + (isDeprecated ? "}" : "");
 +    }
 +
 +    @Override
 +    void propertyTypeDescriptions() {
 +      for (PropertyType type : PropertyType.values()) {
 +        if (type == PropertyType.PREFIX)
 +          continue;
 +        doc.println("\\subsection{" + sanitize(type.toString()) + "}");
 +        doc.println(sanitize(type.getFormatDescription()));
 +        doc.println();
 +      }
 +    }
 +
 +    @Override
 +    String sanitize(String str) {
 +      str = str.replace("\\", "\\textbackslash{}");
 +      str = str.replace("~", "\\textasciitilde{}");
 +      str = str.replace("^", "\\textasciicircum");
 +
 +      str = str.replace("&", "\\&");
 +      str = str.replace("%", "\\%");
 +      str = str.replace("$", "\\$");
 +      str = str.replace("#", "\\#");
 +      str = str.replace("_", "\\_");
 +      str = str.replace("{", "\\{");
 +      str = str.replace("}", "\\}");
 +
 +      str = str.replaceAll("(?:\r\n|\r|\n)", "\\\\\\\\\n");
 +      return str;
 +    }
 +  }
 +
 +  private static Logger log = Logger.getLogger(ConfigurationDocGen.class);
 +
 +  private PrintStream doc;
 +
 +  private final ArrayList<Property> prefixes;
 +
 +  private final TreeMap<String,Property> sortedProps;
 +
 +  ConfigurationDocGen(PrintStream doc) {
 +    this.doc = doc;
 +    this.prefixes = new ArrayList<Property>();
 +    this.sortedProps = new TreeMap<String,Property>();
 +
 +    for (Property prop : Property.values()) {
 +      if (prop.isExperimental())
 +        continue;
 +
 +      if (prop.getType() == PropertyType.PREFIX)
 +        this.prefixes.add(prop);
 +      else
 +        this.sortedProps.put(prop.getKey(), prop);
 +    }
 +  }
 +
 +  private void appendResource(String resourceName) {
 +    InputStream data = ConfigurationDocGen.class.getResourceAsStream(resourceName);
 +    if (data != null) {
 +      byte[] buffer = new byte[1024];
 +      int n;
 +      try {
 +        while ((n = data.read(buffer)) > 0)
-           doc.print(new String(buffer, 0, n, Constants.UTF8));
++          doc.print(new String(buffer, 0, n, UTF_8));
 +      } catch (IOException e) {
 +        e.printStackTrace();
 +        return;
 +      } finally {
 +        try {
 +          data.close();
 +        } catch (IOException ex) {
 +          log.error(ex, ex);
 +        }
 +      }
 +    }
 +  }
 +
 +  private String isZooKeeperMutable(Property prop) {
 +    if (!Property.isValidZooPropertyKey(prop.getKey()))
 +      return "no";
 +    if (Property.isFixedZooPropertyKey(prop))
 +      return "yes but requires restart of the " + prop.getKey().split("[.]")[0];
 +    return "yes";
 +  }
 +
 +  void generateHtml() {
 +    new HTML().generate();
 +  }
 +
 +  void generateLaTeX() {
 +    new LaTeX().generate();
 +  }
 +
 +  /*
 +   * Generate documentation for conf/accumulo-site.xml file usage
 +   */
 +  public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
 +    if (args.length == 2 && args[0].equals("--generate-html")) {
-       new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateHtml();
++      new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateHtml();
 +    } else if (args.length == 2 && args[0].equals("--generate-latex")) {
-       new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateLaTeX();
++      new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateLaTeX();
 +    } else {
 +      throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index ca769cb,7c4f66e..437590f
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@@ -16,11 -16,10 +16,11 @@@
   */
  package org.apache.accumulo.core.data;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.io.Serializable;
 +import java.nio.ByteBuffer;
  
- import org.apache.accumulo.core.Constants;
- 
  public class ArrayByteSequence extends ByteSequence implements Serializable {
    
    private static final long serialVersionUID = 1L;
@@@ -48,22 -47,9 +48,22 @@@
    }
    
    public ArrayByteSequence(String s) {
-     this(s.getBytes(Constants.UTF8));
+     this(s.getBytes(UTF_8));
    }
    
 +  public ArrayByteSequence(ByteBuffer buffer) {
 +    this.length = buffer.remaining();
 +
 +    if (buffer.hasArray()) {
 +      this.data = buffer.array();
 +      this.offset = buffer.position();
 +    } else {
 +      this.data = new byte[length];
 +      this.offset = 0;
 +      buffer.get(data);
 +    }
 +  }
 +
    @Override
    public byte byteAt(int i) {
      


[30/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
index 1226806,0000000..10cf224
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
@@@ -1,617 -1,0 +1,619 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.io.OutputStreamWriter;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.concurrent.Callable;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Future;
 +import java.util.concurrent.ThreadPoolExecutor;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.ServerClient;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.SimpleThreadPool;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
 +import org.apache.accumulo.trace.instrument.TraceExecutorService;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.MapFile;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +
 +/*
 + * Bulk import makes requests of tablet servers, and those requests can take a
 + * long time. Our communications to the tablet server may fail, so we won't know
 + * the status of the request. The master will repeat failed requests so now
 + * there are multiple requests to the tablet server. The tablet server will not
 + * execute the request multiple times, so long as the marker it wrote in the
 + * metadata table stays there. The master needs to know when all requests have
 + * finished so it can remove the markers. Did it start? Did it finish? We can see
 + * that *a* request completed by seeing the flag written into the metadata
 + * table, but we won't know if some other rogue thread is still waiting to start
 + * a thread and repeat the operation.
 + * 
 + * The master can ask the tablet server if it has any requests still running.
 + * Except the tablet server might have some thread about to start a request, but
 + * before it has made any bookkeeping about the request. To prevent problems
 + * like this, an Arbitrator is used. Before starting any new request, the tablet
 + * server checks the Arbitrator to see if the request is still valid.
 + * 
 + */
 +
 +public class BulkImport extends MasterRepo {
 +  public static final String FAILURES_TXT = "failures.txt";
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private static final Logger log = Logger.getLogger(BulkImport.class);
 +
 +  private String tableId;
 +  private String sourceDir;
 +  private String errorDir;
 +  private boolean setTime;
 +
 +  public BulkImport(String tableId, String sourceDir, String errorDir, boolean setTime) {
 +    this.tableId = tableId;
 +    this.sourceDir = sourceDir;
 +    this.errorDir = errorDir;
 +    this.setTime = setTime;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    if (!Utils.getReadLock(tableId, tid).tryLock())
 +      return 100;
 +
 +    Instance instance = HdfsZooInstance.getInstance();
 +    Tables.clearCache(instance);
 +    if (Tables.getTableState(instance, tableId) == TableState.ONLINE) {
 +      long reserve1, reserve2;
 +      reserve1 = reserve2 = Utils.reserveHdfsDirectory(sourceDir, tid);
 +      if (reserve1 == 0)
 +        reserve2 = Utils.reserveHdfsDirectory(errorDir, tid);
 +      return reserve2;
 +    } else {
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.OFFLINE, null);
 +    }
 +  }
 +
 +  @Override
 +  //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
 +  @SuppressWarnings("deprecation")
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    log.debug(" tid " + tid + " sourceDir " + sourceDir);
 +
 +    Utils.getReadLock(tableId, tid).lock();
 +
 +    // check that the error directory exists and is empty
 +    VolumeManager fs = master.getFileSystem();
 +
 +    Path errorPath = new Path(errorDir);
 +    FileStatus errorStatus = null;
 +    try {
 +      errorStatus = fs.getFileStatus(errorPath);
 +    } catch (FileNotFoundException ex) {
 +      // ignored
 +    }
 +    if (errorStatus == null)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
 +          + " does not exist");
 +    if (!errorStatus.isDir())
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
 +          + " is not a directory");
 +    if (fs.listStatus(errorPath).length != 0)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
 +          + " is not empty");
 +
 +    ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
 +
 +    // move the files into the directory
 +    try {
 +      String bulkDir = prepareBulkImport(fs, sourceDir, tableId);
 +      log.debug(" tid " + tid + " bulkDir " + bulkDir);
 +      return new LoadFiles(tableId, sourceDir, bulkDir, errorDir, setTime);
 +    } catch (IOException ex) {
 +      log.error("error preparing the bulk import directory", ex);
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_INPUT_DIRECTORY, sourceDir + ": "
 +          + ex);
 +    }
 +  }
 +
 +  private Path createNewBulkDir(VolumeManager fs, String tableId) throws IOException {
 +    Path tempPath = fs.matchingFileSystem(new Path(sourceDir), ServerConstants.getTablesDirs());
 +    if (tempPath == null)
 +      throw new IllegalStateException(sourceDir + " is not in a known namespace");
 +
 +    String tableDir = tempPath.toString();
 +    if (tableDir == null)
 +      throw new IllegalStateException(sourceDir + " is not in a known namespace");
 +    Path directory = new Path(tableDir + "/" + tableId);
 +    fs.mkdirs(directory);
 +
 +    // only one should be able to create the lock file
 +    // the purpose of the lock file is to avoid a race
 +    // condition between the call to fs.exists() and
 +    // fs.mkdirs()... if only hadoop had a mkdir() function
 +    // that failed when the dir existed
 +
 +    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
 +    while (true) {
 +      Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
 +      if (fs.exists(newBulkDir)) // sanity check
 +        throw new IllegalStateException("Dir exist when it should not " + newBulkDir);
 +      if (fs.mkdirs(newBulkDir))
 +        return newBulkDir;
 +      log.warn("Failed to create " + newBulkDir + " for unknown reason");
 +
 +      UtilWaitThread.sleep(3000);
 +    }
 +  }
 +
 +  //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
 +  @SuppressWarnings("deprecation")
 +  private String prepareBulkImport(VolumeManager fs, String dir, String tableId) throws IOException {
 +    Path bulkDir = createNewBulkDir(fs, tableId);
 +
 +    MetadataTableUtil.addBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
 +
 +    Path dirPath = new Path(dir);
 +    FileStatus[] mapFiles = fs.listStatus(dirPath);
 +
 +    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
 +    for (FileStatus fileStatus : mapFiles) {
 +      String sa[] = fileStatus.getPath().getName().split("\\.");
 +      String extension = "";
 +      if (sa.length > 1) {
 +        extension = sa[sa.length - 1];
 +
 +        if (!FileOperations.getValidExtensions().contains(extension)) {
 +          log.warn(fileStatus.getPath() + " does not have a valid extension, ignoring");
 +          continue;
 +        }
 +      } else {
 +        // assume it is a map file
 +        extension = Constants.MAPFILE_EXTENSION;
 +      }
 +
 +      if (extension.equals(Constants.MAPFILE_EXTENSION)) {
 +        if (!fileStatus.isDir()) {
 +          log.warn(fileStatus.getPath() + " is not a map file, ignoring");
 +          continue;
 +        }
 +
 +        if (fileStatus.getPath().getName().equals("_logs")) {
 +          log.info(fileStatus.getPath() + " is probably a log directory from a map/reduce task, skipping");
 +          continue;
 +        }
 +        try {
 +          FileStatus dataStatus = fs.getFileStatus(new Path(fileStatus.getPath(), MapFile.DATA_FILE_NAME));
 +          if (dataStatus.isDir()) {
 +            log.warn(fileStatus.getPath() + " is not a map file, ignoring");
 +            continue;
 +          }
 +        } catch (FileNotFoundException fnfe) {
 +          log.warn(fileStatus.getPath() + " is not a map file, ignoring");
 +          continue;
 +        }
 +      }
 +
 +      String newName = "I" + namer.getNextName() + "." + extension;
 +      Path newPath = new Path(bulkDir, newName);
 +      try {
 +        fs.rename(fileStatus.getPath(), newPath);
 +        log.debug("Moved " + fileStatus.getPath() + " to " + newPath);
 +      } catch (IOException E1) {
 +        log.error("Could not move: " + fileStatus.getPath().toString() + " " + E1.getMessage());
 +      }
 +    }
 +    return bulkDir.toString();
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    // unreserve source/error directories
 +    Utils.unreserveHdfsDirectory(sourceDir, tid);
 +    Utils.unreserveHdfsDirectory(errorDir, tid);
 +    Utils.getReadLock(tableId, tid).unlock();
 +  }
 +}
 +
 +class CleanUpBulkImport extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private static final Logger log = Logger.getLogger(CleanUpBulkImport.class);
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String error;
 +
 +  public CleanUpBulkImport(String tableId, String source, String bulk, String error) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.error = error;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    log.debug("removing the bulk processing flag file in " + bulk);
 +    Path bulkDir = new Path(bulk);
 +    MetadataTableUtil.removeBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
 +    MetadataTableUtil.addDeleteEntry(tableId, bulkDir.toString());
 +    log.debug("removing the metadata table markers for loaded files");
 +    Connector conn = master.getConnector();
 +    MetadataTableUtil.removeBulkLoadEntries(conn, tableId, tid);
 +    log.debug("releasing HDFS reservations for " + source + " and " + error);
 +    Utils.unreserveHdfsDirectory(source, tid);
 +    Utils.unreserveHdfsDirectory(error, tid);
 +    Utils.getReadLock(tableId, tid).unlock();
 +    log.debug("completing bulk import transaction " + tid);
 +    ZooArbitrator.cleanup(Constants.BULK_ARBITRATOR_TYPE, tid);
 +    return null;
 +  }
 +}
 +
 +class CompleteBulkImport extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String error;
 +
 +  public CompleteBulkImport(String tableId, String source, String bulk, String error) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.error = error;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    ZooArbitrator.stop(Constants.BULK_ARBITRATOR_TYPE, tid);
 +    return new CopyFailed(tableId, source, bulk, error);
 +  }
 +}
 +
 +class CopyFailed extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String error;
 +
 +  public CopyFailed(String tableId, String source, String bulk, String error) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.error = error;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    Set<TServerInstance> finished = new HashSet<TServerInstance>();
 +    Set<TServerInstance> running = master.onlineTabletServers();
 +    for (TServerInstance server : running) {
 +      try {
 +        TServerConnection client = master.getConnection(server);
 +        if (client != null && !client.isActive(tid))
 +          finished.add(server);
 +      } catch (TException ex) {
 +        log.info("Ignoring error trying to check on tid " + tid + " from server " + server + ": " + ex);
 +      }
 +    }
 +    if (finished.containsAll(running))
 +      return 0;
 +    return 500;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    // This needs to execute after the arbiter is stopped
 +
 +    VolumeManager fs = master.getFileSystem();
 +
 +    if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
 +      return new CleanUpBulkImport(tableId, source, bulk, error);
 +
 +    HashMap<FileRef,String> failures = new HashMap<FileRef,String>();
 +    HashMap<FileRef,String> loadedFailures = new HashMap<FileRef,String>();
 +
 +    FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
-     BufferedReader in = new BufferedReader(new InputStreamReader(failFile, Constants.UTF8));
++    BufferedReader in = new BufferedReader(new InputStreamReader(failFile, UTF_8));
 +    try {
 +      String line = null;
 +      while ((line = in.readLine()) != null) {
 +        Path path = new Path(line);
 +        if (!fs.exists(new Path(error, path.getName())))
 +          failures.put(new FileRef(line, path), line);
 +      }
 +    } finally {
 +      failFile.close();
 +    }
 +
 +    /*
 +     * I thought I could move files that have no file references in the table. However its possible a clone references a file. Therefore only move files that
 +     * have no loaded markers.
 +     */
 +
 +    // determine which failed files were loaded
 +    Connector conn = master.getConnector();
 +    Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +
 +    for (Entry<Key,Value> entry : mscanner) {
 +      if (Long.parseLong(entry.getValue().toString()) == tid) {
 +        FileRef loadedFile = new FileRef(fs, entry.getKey());
 +        String absPath = failures.remove(loadedFile);
 +        if (absPath != null) {
 +          loadedFailures.put(loadedFile, absPath);
 +        }
 +      }
 +    }
 +
 +    // move failed files that were not loaded
 +    for (String failure : failures.values()) {
 +      Path orig = new Path(failure);
 +      Path dest = new Path(error, orig.getName());
 +      fs.rename(orig, dest);
 +      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
 +    }
 +
 +    if (loadedFailures.size() > 0) {
 +      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
 +          + Constants.ZBULK_FAILED_COPYQ);
 +
 +      HashSet<String> workIds = new HashSet<String>();
 +
 +      for (String failure : loadedFailures.values()) {
 +        Path orig = new Path(failure);
 +        Path dest = new Path(error, orig.getName());
 +
 +        if (fs.exists(dest))
 +          continue;
 +
-         bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
++        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8));
 +        workIds.add(orig.getName());
 +        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
 +      }
 +
 +      bifCopyQueue.waitUntilDone(workIds);
 +    }
 +
 +    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
 +    return new CleanUpBulkImport(tableId, source, bulk, error);
 +  }
 +
 +}
 +
 +class LoadFiles extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private static ExecutorService threadPool = null;
 +  private static final Logger log = Logger.getLogger(BulkImport.class);
 +
 +  private String tableId;
 +  private String source;
 +  private String bulk;
 +  private String errorDir;
 +  private boolean setTime;
 +
 +  public LoadFiles(String tableId, String source, String bulk, String errorDir, boolean setTime) {
 +    this.tableId = tableId;
 +    this.source = source;
 +    this.bulk = bulk;
 +    this.errorDir = errorDir;
 +    this.setTime = setTime;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    if (master.onlineTabletServers().size() == 0)
 +      return 500;
 +    return 0;
 +  }
 +
 +  private static synchronized ExecutorService getThreadPool(Master master) {
 +    if (threadPool == null) {
 +      int threadPoolSize = master.getSystemConfiguration().getCount(Property.MASTER_BULK_THREADPOOL_SIZE);
 +      ThreadPoolExecutor pool = new SimpleThreadPool(threadPoolSize, "bulk import");
 +      pool.allowCoreThreadTimeOut(true);
 +      threadPool = new TraceExecutorService(pool);
 +    }
 +    return threadPool;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(final long tid, final Master master) throws Exception {
 +    ExecutorService executor = getThreadPool(master);
 +    final SiteConfiguration conf = ServerConfiguration.getSiteConfiguration();
 +    VolumeManager fs = master.getFileSystem();
 +    List<FileStatus> files = new ArrayList<FileStatus>();
 +    for (FileStatus entry : fs.listStatus(new Path(bulk))) {
 +      files.add(entry);
 +    }
 +    log.debug("tid " + tid + " importing " + files.size() + " files");
 +
 +    Path writable = new Path(this.errorDir, ".iswritable");
 +    if (!fs.createNewFile(writable)) {
 +      // Maybe this is a re-try... clear the flag and try again
 +      fs.delete(writable);
 +      if (!fs.createNewFile(writable))
 +        throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY,
 +            "Unable to write to " + this.errorDir);
 +    }
 +    fs.delete(writable);
 +
 +    final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<String>());
 +    for (FileStatus f : files)
 +      filesToLoad.add(f.getPath().toString());
 +
 +    final int RETRIES = Math.max(1, conf.getCount(Property.MASTER_BULK_RETRIES));
 +    for (int attempt = 0; attempt < RETRIES && filesToLoad.size() > 0; attempt++) {
 +      List<Future<List<String>>> results = new ArrayList<Future<List<String>>>();
 +
 +      if (master.onlineTabletServers().size() == 0)
 +        log.warn("There are no tablet server to process bulk import, waiting (tid = " + tid + ")");
 +
 +      while (master.onlineTabletServers().size() == 0) {
 +        UtilWaitThread.sleep(500);
 +      }
 +
 +      // Use the threadpool to assign files one-at-a-time to the server
 +      final List<String> loaded = Collections.synchronizedList(new ArrayList<String>());
 +      for (final String file : filesToLoad) {
 +        results.add(executor.submit(new Callable<List<String>>() {
 +          @Override
 +          public List<String> call() {
 +            List<String> failures = new ArrayList<String>();
 +            ClientService.Client client = null;
 +            String server = null;
 +            try {
 +              // get a connection to a random tablet server, do not prefer cached connections because
 +              // this is running on the master and there are lots of connections to tablet servers
 +              // serving the metadata tablets
 +              long timeInMillis = master.getConfiguration().getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
 +              Pair<String,Client> pair = ServerClient.getConnection(master.getInstance(), false, timeInMillis);
 +              client = pair.getSecond();
 +              server = pair.getFirst();
 +              List<String> attempt = Collections.singletonList(file);
 +              log.debug("Asking " + pair.getFirst() + " to bulk import " + file);
 +              List<String> fail = client.bulkImportFiles(Tracer.traceInfo(), SystemCredentials.get().toThrift(master.getInstance()), tid, tableId, attempt,
 +                  errorDir, setTime);
 +              if (fail.isEmpty()) {
 +                loaded.add(file);
 +              } else {
 +                failures.addAll(fail);
 +              }
 +            } catch (Exception ex) {
 +              log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
 +            } finally {
 +              ServerClient.close(client);
 +            }
 +            return failures;
 +          }
 +        }));
 +      }
 +      Set<String> failures = new HashSet<String>();
 +      for (Future<List<String>> f : results)
 +        failures.addAll(f.get());
 +      filesToLoad.removeAll(loaded);
 +      if (filesToLoad.size() > 0) {
 +        log.debug("tid " + tid + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
 +        UtilWaitThread.sleep(100);
 +      }
 +    }
 +
 +    FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
-     BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, Constants.UTF8));
++    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8));
 +    try {
 +      for (String f : filesToLoad) {
 +        out.write(f);
 +        out.write("\n");
 +      }
 +    } finally {
 +      out.close();
 +    }
 +
 +    // return the next step, which will perform cleanup
 +    return new CompleteBulkImport(tableId, source, bulk, errorDir);
 +  }
 +
 +  static String sampleList(Collection<?> potentiallyLongList, int max) {
 +    StringBuffer result = new StringBuffer();
 +    result.append("[");
 +    int i = 0;
 +    for (Object obj : potentiallyLongList) {
 +      result.append(obj);
 +      if (i >= max) {
 +        result.append("...");
 +        break;
 +      } else {
 +        result.append(", ");
 +      }
 +      i++;
 +    }
 +    if (i < max)
 +      result.delete(result.length() - 2, result.length());
 +    result.append("]");
 +    return result.toString();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
index ebceaa0,0000000..fd2441c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
@@@ -1,105 -1,0 +1,107 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +class FinishCancelCompaction extends MasterRepo {
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +
 +  public FinishCancelCompaction(String tableId) {
 +    this.tableId = tableId;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +    Utils.getReadLock(tableId, tid).unlock();
 +    return null;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +
 +  }
 +}
 +
 +/**
 + *
 + */
 +public class CancelCompactions extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +  private String namespaceId;
 +
 +  public CancelCompactions(String tableId) {
 +    this.tableId = tableId;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT_CANCEL)
 +        + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT_CANCEL);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +    String zCompactID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
 +    String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    byte[] currentValue = zoo.getData(zCompactID, null);
 +
-     String cvs = new String(currentValue, Constants.UTF8);
++    String cvs = new String(currentValue, UTF_8);
 +    String[] tokens = cvs.split(",");
 +    final long flushID = Long.parseLong(tokens[0]);
 +
 +    zoo.mutate(zCancelID, null, null, new Mutator() {
 +      @Override
 +      public byte[] mutate(byte[] currentValue) throws Exception {
-         long cid = Long.parseLong(new String(currentValue, Constants.UTF8));
++        long cid = Long.parseLong(new String(currentValue, UTF_8));
 +
 +        if (cid < flushID)
-           return Long.toString(flushID).getBytes(Constants.UTF8);
++          return Long.toString(flushID).getBytes(UTF_8);
 +        else
-           return Long.toString(cid).getBytes(Constants.UTF8);
++          return Long.toString(cid).getBytes(UTF_8);
 +      }
 +    });
 +
 +    return new FinishCancelCompaction(tableId);
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    Utils.unreserveNamespace(namespaceId, tid, false);
 +    Utils.unreserveTable(tableId, tid, false);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index a643ccb,0000000..fbaa70d
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@@ -1,400 -1,0 +1,402 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.DataInput;
 +import java.io.DataOutput;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.commons.codec.binary.Hex;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.io.WritableUtils;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +class CompactionDriver extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private long compactId;
 +  private String tableId;
 +  private byte[] startRow;
 +  private byte[] endRow;
 +  private String namespaceId;
 +
 +  public CompactionDriver(long compactId, String tableId, byte[] startRow, byte[] endRow) {
 +
 +    this.compactId = compactId;
 +    this.tableId = tableId;
 +    this.startRow = startRow;
 +    this.endRow = endRow;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +
 +    String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    if (Long.parseLong(new String(zoo.getData(zCancelID, null))) >= compactId) {
 +      // compaction was canceled
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, "Compaction canceled");
 +    }
 +
 +    MapCounter<TServerInstance> serversToFlush = new MapCounter<TServerInstance>();
 +    Connector conn = master.getConnector();
 +
 +    Scanner scanner;
 +
 +    if (tableId.equals(MetadataTable.ID)) {
 +      scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
 +      scanner.setRange(MetadataSchema.TabletsSection.getRange());
 +    } else {
 +      scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 +      Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange();
 +      scanner.setRange(range);
 +    }
 +
 +    TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner);
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
 +    scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +
 +    long t1 = System.currentTimeMillis();
 +    RowIterator ri = new RowIterator(scanner);
 +
 +    int tabletsToWaitFor = 0;
 +    int tabletCount = 0;
 +
 +    while (ri.hasNext()) {
 +      Iterator<Entry<Key,Value>> row = ri.next();
 +      long tabletCompactID = -1;
 +
 +      TServerInstance server = null;
 +
 +      Entry<Key,Value> entry = null;
 +      while (row.hasNext()) {
 +        entry = row.next();
 +        Key key = entry.getKey();
 +
 +        if (TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +          tabletCompactID = Long.parseLong(entry.getValue().toString());
 +
 +        if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily()))
 +          server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
 +      }
 +
 +      if (tabletCompactID < compactId) {
 +        tabletsToWaitFor++;
 +        if (server != null)
 +          serversToFlush.increment(server, 1);
 +      }
 +
 +      tabletCount++;
 +
 +      Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
 +      if (tabletEndRow == null || (endRow != null && tabletEndRow.compareTo(new Text(endRow)) >= 0))
 +        break;
 +    }
 +
 +    long scanTime = System.currentTimeMillis() - t1;
 +
 +    Instance instance = master.getInstance();
 +    Tables.clearCache(instance);
 +    if (tabletCount == 0 && !Tables.exists(instance, tableId))
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
 +
 +    if (serversToFlush.size() == 0 && Tables.getTableState(instance, tableId) == TableState.OFFLINE)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
 +
 +    if (tabletsToWaitFor == 0)
 +      return 0;
 +
 +    for (TServerInstance tsi : serversToFlush.keySet()) {
 +      try {
 +        final TServerConnection server = master.getConnection(tsi);
 +        if (server != null)
 +          server.compact(master.getMasterLock(), tableId, startRow, endRow);
 +      } catch (TException ex) {
 +        Logger.getLogger(CompactionDriver.class).error(ex.toString());
 +      }
 +    }
 +
 +    long sleepTime = 500;
 +
 +    if (serversToFlush.size() > 0)
 +      sleepTime = Collections.max(serversToFlush.values()) * sleepTime; // make wait time depend on the server with the most to
 +                                                                        // compact
 +
 +    sleepTime = Math.max(2 * scanTime, sleepTime);
 +
 +    sleepTime = Math.min(sleepTime, 30000);
 +
 +    return sleepTime;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +    CompactRange.removeIterators(tid, tableId);
 +    Utils.getReadLock(tableId, tid).unlock();
 +    Utils.getReadLock(namespaceId, tid).unlock();
 +    return null;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +
 +  }
 +
 +}
 +
 +public class CompactRange extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +  private byte[] startRow;
 +  private byte[] endRow;
 +  private byte[] iterators;
 +  private String namespaceId;
 +
 +  public static class CompactionIterators implements Writable {
 +    byte[] startRow;
 +    byte[] endRow;
 +    List<IteratorSetting> iterators;
 +
 +    public CompactionIterators(byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) {
 +      this.startRow = startRow;
 +      this.endRow = endRow;
 +      this.iterators = iterators;
 +    }
 +
 +    public CompactionIterators() {
 +      startRow = null;
 +      endRow = null;
 +      iterators = Collections.emptyList();
 +    }
 +
 +    @Override
 +    public void write(DataOutput out) throws IOException {
 +      out.writeBoolean(startRow != null);
 +      if (startRow != null) {
 +        out.writeInt(startRow.length);
 +        out.write(startRow);
 +      }
 +
 +      out.writeBoolean(endRow != null);
 +      if (endRow != null) {
 +        out.writeInt(endRow.length);
 +        out.write(endRow);
 +      }
 +
 +      out.writeInt(iterators.size());
 +      for (IteratorSetting is : iterators) {
 +        is.write(out);
 +      }
 +    }
 +
 +    @Override
 +    public void readFields(DataInput in) throws IOException {
 +      if (in.readBoolean()) {
 +        startRow = new byte[in.readInt()];
 +        in.readFully(startRow);
 +      } else {
 +        startRow = null;
 +      }
 +
 +      if (in.readBoolean()) {
 +        endRow = new byte[in.readInt()];
 +        in.readFully(endRow);
 +      } else {
 +        endRow = null;
 +      }
 +
 +      int num = in.readInt();
 +      iterators = new ArrayList<IteratorSetting>(num);
 +
 +      for (int i = 0; i < num; i++) {
 +        iterators.add(new IteratorSetting(in));
 +      }
 +    }
 +
 +    public Text getEndRow() {
 +      if (endRow == null)
 +        return null;
 +      return new Text(endRow);
 +    }
 +
 +    public Text getStartRow() {
 +      if (startRow == null)
 +        return null;
 +      return new Text(startRow);
 +    }
 +
 +    public List<IteratorSetting> getIterators() {
 +      return iterators;
 +    }
 +  }
 +
 +  public CompactRange(String tableId, byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) throws ThriftTableOperationException {
 +    this.tableId = tableId;
 +    this.startRow = startRow.length == 0 ? null : startRow;
 +    this.endRow = endRow.length == 0 ? null : endRow;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +
 +    if (iterators.size() > 0) {
 +      this.iterators = WritableUtils.toByteArray(new CompactionIterators(this.startRow, this.endRow, iterators));
 +    } else {
 +      iterators = null;
 +    }
 +
 +    if (this.startRow != null && this.endRow != null && new Text(startRow).compareTo(new Text(endRow)) >= 0)
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.BAD_RANGE,
 +          "start row must be less than end row");
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT)
 +        + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(final long tid, Master environment) throws Exception {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    byte[] cid;
 +    try {
 +      cid = zoo.mutate(zTablePath, null, null, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] currentValue) throws Exception {
-           String cvs = new String(currentValue, Constants.UTF8);
++          String cvs = new String(currentValue, UTF_8);
 +          String[] tokens = cvs.split(",");
 +          long flushID = Long.parseLong(tokens[0]);
 +          flushID++;
 +
 +          String txidString = String.format("%016x", tid);
 +
 +          for (int i = 1; i < tokens.length; i++) {
 +            if (tokens[i].startsWith(txidString))
 +              continue; // skip self
 +
 +            throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER,
 +                "Another compaction with iterators is running");
 +          }
 +
 +          StringBuilder encodedIterators = new StringBuilder();
 +
 +          if (iterators != null) {
 +            Hex hex = new Hex();
 +            encodedIterators.append(",");
 +            encodedIterators.append(txidString);
 +            encodedIterators.append("=");
-             encodedIterators.append(new String(hex.encode(iterators), Constants.UTF8));
++            encodedIterators.append(new String(hex.encode(iterators), UTF_8));
 +          }
 +
-           return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++          return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
 +        }
 +      });
 +
-       return new CompactionDriver(Long.parseLong(new String(cid, Constants.UTF8).split(",")[0]), tableId, startRow, endRow);
++      return new CompactionDriver(Long.parseLong(new String(cid, UTF_8).split(",")[0]), tableId, startRow, endRow);
 +    } catch (NoNodeException nne) {
 +      throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
 +    }
 +
 +  }
 +
 +  static void removeIterators(final long txid, String tableId) throws Exception {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    zoo.mutate(zTablePath, null, null, new Mutator() {
 +      @Override
 +      public byte[] mutate(byte[] currentValue) throws Exception {
-         String cvs = new String(currentValue, Constants.UTF8);
++        String cvs = new String(currentValue, UTF_8);
 +        String[] tokens = cvs.split(",");
 +        long flushID = Long.parseLong(tokens[0]);
 +
 +        String txidString = String.format("%016x", txid);
 +
 +        StringBuilder encodedIterators = new StringBuilder();
 +        for (int i = 1; i < tokens.length; i++) {
 +          if (tokens[i].startsWith(txidString))
 +            continue;
 +          encodedIterators.append(",");
 +          encodedIterators.append(tokens[i]);
 +        }
 +
-         return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++        return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
 +      }
 +    });
 +
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    try {
 +      removeIterators(tid, tableId);
 +    } finally {
 +      Utils.unreserveNamespace(namespaceId, tid, false);
 +      Utils.unreserveTable(tableId, tid, false);
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index eeb9b16,0000000..037455c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@@ -1,316 -1,0 +1,318 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedOutputStream;
 +import java.io.BufferedWriter;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.io.OutputStreamWriter;
 +import java.io.Serializable;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.zip.ZipEntry;
 +import java.util.zip.ZipOutputStream;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +
 +class ExportInfo implements Serializable {
 +  
 +  private static final long serialVersionUID = 1L;
 +  
 +  public String tableName;
 +  public String tableID;
 +  public String exportDir;
 +  public String namespaceID;
 +}
 +
 +class WriteExportFiles extends MasterRepo {
 +  
 +  private static final long serialVersionUID = 1L;
 +  private final ExportInfo tableInfo;
 +  
 +  WriteExportFiles(ExportInfo tableInfo) {
 +    this.tableInfo = tableInfo;
 +  }
 +  
 +  private void checkOffline(Connector conn) throws Exception {
 +    if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
 +      Tables.clearCache(conn.getInstance());
 +      if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
 +        throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
 +            "Table is not offline");
 +      }
 +    }
 +  }
 +  
 +  @Override
 +  public long isReady(long tid, Master master) throws Exception {
 +    
 +    long reserved = Utils.reserveNamespace(tableInfo.namespaceID, tid, false, true, TableOperation.EXPORT)
 +        + Utils.reserveTable(tableInfo.tableID, tid, false, true, TableOperation.EXPORT);
 +    if (reserved > 0)
 +      return reserved;
 +    
 +    Connector conn = master.getConnector();
 +    
 +    checkOffline(conn);
 +    
 +    Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    metaScanner.setRange(new KeyExtent(new Text(tableInfo.tableID), null, null).toMetadataRange());
 +    
 +    // scan for locations
 +    metaScanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +    metaScanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
 +    
 +    if (metaScanner.iterator().hasNext()) {
 +      return 500;
 +    }
 +    
 +    // use the same range to check for walogs that we used to check for hosted (or future hosted) tablets
 +    // this is done as a separate scan after we check for locations, because walogs are okay only if there is no location
 +    metaScanner.clearColumns();
 +    metaScanner.fetchColumnFamily(LogColumnFamily.NAME);
 +    
 +    if (metaScanner.iterator().hasNext()) {
 +      throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
 +          "Write ahead logs found for table");
 +    }
 +    
 +    return 0;
 +  }
 +  
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    Connector conn = master.getConnector();
 +    
 +    try {
 +      exportTable(master.getFileSystem(), conn, tableInfo.tableName, tableInfo.tableID, tableInfo.exportDir);
 +    } catch (IOException ioe) {
 +      throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
 +          "Failed to create export files " + ioe.getMessage());
 +    }
 +    Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
 +    Utils.unreserveTable(tableInfo.tableID, tid, false);
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +    return null;
 +  }
 +  
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
 +    Utils.unreserveTable(tableInfo.tableID, tid, false);
 +  }
 +  
 +  public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
 +    
 +    fs.mkdirs(new Path(exportDir));
 +    Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
 +    
 +    FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
 +    ZipOutputStream zipOut = new ZipOutputStream(fileOut);
 +    BufferedOutputStream bufOut = new BufferedOutputStream(zipOut);
 +    DataOutputStream dataOut = new DataOutputStream(bufOut);
 +    
 +    try {
 +      
 +      zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_INFO_FILE));
-       OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++      OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
 +      osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.VERSION + "\n");
 +      osw.append("srcInstanceName:" + conn.getInstance().getInstanceName() + "\n");
 +      osw.append("srcInstanceID:" + conn.getInstance().getInstanceID() + "\n");
 +      osw.append("srcZookeepers:" + conn.getInstance().getZooKeepers() + "\n");
 +      osw.append("srcTableName:" + tableName + "\n");
 +      osw.append("srcTableID:" + tableID + "\n");
 +      osw.append(ExportTable.DATA_VERSION_PROP + ":" + ServerConstants.DATA_VERSION + "\n");
 +      osw.append("srcCodeVersion:" + Constants.VERSION + "\n");
 +      
 +      osw.flush();
 +      dataOut.flush();
 +      
 +      exportConfig(conn, tableID, zipOut, dataOut);
 +      dataOut.flush();
 +      
 +      Map<String,String> uniqueFiles = exportMetadata(fs, conn, tableID, zipOut, dataOut);
 +      
 +      dataOut.close();
 +      dataOut = null;
 +      
 +      createDistcpFile(fs, exportDir, exportMetaFilePath, uniqueFiles);
 +      
 +    } finally {
 +      if (dataOut != null)
 +        dataOut.close();
 +    }
 +  }
 +  
 +  private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException {
-     BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), Constants.UTF8));
++    BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), UTF_8));
 +    
 +    try {
 +      for (String file : uniqueFiles.values()) {
 +        distcpOut.append(file);
 +        distcpOut.newLine();
 +      }
 +      
 +      distcpOut.append(exportMetaFilePath.toString());
 +      distcpOut.newLine();
 +      
 +      distcpOut.close();
 +      distcpOut = null;
 +      
 +    } finally {
 +      if (distcpOut != null)
 +        distcpOut.close();
 +    }
 +  }
 +  
 +  private static Map<String,String> exportMetadata(VolumeManager fs, Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut)
 +      throws IOException, TableNotFoundException {
 +    zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_METADATA_FILE));
 +    
 +    Map<String,String> uniqueFiles = new HashMap<String,String>();
 +    
 +    Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    metaScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(metaScanner);
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(metaScanner);
 +    metaScanner.setRange(new KeyExtent(new Text(tableID), null, null).toMetadataRange());
 +    
 +    for (Entry<Key,Value> entry : metaScanner) {
 +      entry.getKey().write(dataOut);
 +      entry.getValue().write(dataOut);
 +      
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        String path = fs.getFullPath(entry.getKey()).toString();
 +        String tokens[] = path.split("/");
 +        if (tokens.length < 1) {
 +          throw new RuntimeException("Illegal path " + path);
 +        }
 +        
 +        String filename = tokens[tokens.length - 1];
 +        
 +        String existingPath = uniqueFiles.get(filename);
 +        if (existingPath == null) {
 +          uniqueFiles.put(filename, path);
 +        } else if (!existingPath.equals(path)) {
 +          // make sure file names are unique, should only apply for tables with file names generated by Accumulo 1.3 and earlier
 +          throw new IOException("Cannot export table with nonunique file names " + filename + ". Major compact table.");
 +        }
 +        
 +      }
 +    }
 +    return uniqueFiles;
 +  }
 +  
 +  private static void exportConfig(Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws AccumuloException,
 +      AccumuloSecurityException, TableNotFoundException, IOException {
 +    
 +    DefaultConfiguration defaultConfig = AccumuloConfiguration.getDefaultConfiguration();
 +    Map<String,String> siteConfig = conn.instanceOperations().getSiteConfiguration();
 +    Map<String,String> systemConfig = conn.instanceOperations().getSystemConfiguration();
 +    
 +    TableConfiguration tableConfig = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableID);
 +    
-     OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++    OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
 +    
 +    // only put props that are different than defaults and higher level configurations
 +    zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));
 +    for (Entry<String,String> prop : tableConfig) {
 +      if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) {
 +        Property key = Property.getPropertyByKey(prop.getKey());
 +        
 +        if (key == null || !defaultConfig.get(key).equals(prop.getValue())) {
 +          if (!prop.getValue().equals(siteConfig.get(prop.getKey())) && !prop.getValue().equals(systemConfig.get(prop.getKey()))) {
 +            osw.append(prop.getKey() + "=" + prop.getValue() + "\n");
 +          }
 +        }
 +      }
 +    }
 +    
 +    osw.flush();
 +  }
 +}
 +
 +public class ExportTable extends MasterRepo {
 +  private static final long serialVersionUID = 1L;
 +  
 +  private final ExportInfo tableInfo;
 +  
 +  public ExportTable(String tableName, String tableId, String exportDir) {
 +    tableInfo = new ExportInfo();
 +    tableInfo.tableName = tableName;
 +    tableInfo.exportDir = exportDir;
 +    tableInfo.tableID = tableId;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    tableInfo.namespaceID = Tables.getNamespaceId(inst, tableId);
 +  }
 +  
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +  }
 +  
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    return new WriteExportFiles(tableInfo);
 +  }
 +  
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +  }
 +  
 +  public static final int VERSION = 1;
 +  
 +  public static final String DATA_VERSION_PROP = "srcDataVersion";
 +  public static final String EXPORT_VERSION_PROP = "exportVersion";
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 8de6bce,0000000..05ec263
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@@ -1,653 -1,0 +1,655 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedInputStream;
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.DataInputStream;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.io.OutputStreamWriter;
 +import java.io.Serializable;
 +import java.util.Arrays;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.zip.ZipEntry;
 +import java.util.zip.ZipInputStream;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.TableOperationsImpl;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.FastFormat;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.TablePropUtil;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +/**
 + *
 + */
 +class ImportedTableInfo implements Serializable {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  public String exportDir;
 +  public String user;
 +  public String tableName;
 +  public String tableId;
 +  public String importDir;
 +  public String namespaceId;
 +}
 +
 +class FinishImportTable extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  public FinishImportTable(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return 0;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +
 +    env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir, "mappings.txt"));
 +
 +    TableManager.getInstance().transitionTableState(tableInfo.tableId, TableState.ONLINE);
 +
 +    Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
 +    Utils.unreserveTable(tableInfo.tableId, tid, true);
 +
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +
 +    env.getEventCoordinator().event("Imported table %s ", tableInfo.tableName);
 +
 +    Logger.getLogger(FinishImportTable.class).debug("Imported table " + tableInfo.tableId + " " + tableInfo.tableName);
 +
 +    return null;
 +  }
 +
 +  @Override
 +  public String getReturn() {
 +    return tableInfo.tableId;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {}
 +
 +}
 +
 +class MoveExportedFiles extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  MoveExportedFiles(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    try {
 +      VolumeManager fs = master.getFileSystem();
 +
 +      Map<String,String> fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo);
 +
 +      for (String oldFileName : fileNameMappings.keySet()) {
 +        if (!fs.exists(new Path(tableInfo.exportDir, oldFileName))) {
 +          throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +              "File referenced by exported table does not exists " + oldFileName);
 +        }
 +      }
 +
 +      FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
 +
 +      for (FileStatus fileStatus : files) {
 +        String newName = fileNameMappings.get(fileStatus.getPath().getName());
 +
 +        if (newName != null)
 +          fs.rename(fileStatus.getPath(), new Path(tableInfo.importDir, newName));
 +      }
 +
 +      return new FinishImportTable(tableInfo);
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error renaming files " + ioe.getMessage());
 +    }
 +  }
 +}
 +
 +class PopulateMetadataTable extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  PopulateMetadataTable(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  static Map<String,String> readMappingFile(VolumeManager fs, ImportedTableInfo tableInfo) throws Exception {
-     BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), Constants.UTF8));
++    BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), UTF_8));
 +
 +    try {
 +      Map<String,String> map = new HashMap<String,String>();
 +
 +      String line = null;
 +      while ((line = in.readLine()) != null) {
 +        String sa[] = line.split(":", 2);
 +        map.put(sa[0], sa[1]);
 +      }
 +
 +      return map;
 +    } finally {
 +      in.close();
 +    }
 +
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +
 +    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
 +
 +    BatchWriter mbw = null;
 +    ZipInputStream zis = null;
 +
 +    try {
 +      VolumeManager fs = master.getFileSystem();
 +
 +      mbw = master.getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +
 +      zis = new ZipInputStream(fs.open(path));
 +
 +      Map<String,String> fileNameMappings = readMappingFile(fs, tableInfo);
 +
 +      log.info("importDir is " + tableInfo.importDir);
 +
 +      // This is a directory already prefixed with proper volume information e.g. hdfs://localhost:8020/path/to/accumulo/tables/...
 +      final String bulkDir = tableInfo.importDir;
 +
 +      final String[] tableDirs = ServerConstants.getTablesDirs();
 +
 +      ZipEntry zipEntry;
 +      while ((zipEntry = zis.getNextEntry()) != null) {
 +        if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) {
 +          DataInputStream in = new DataInputStream(new BufferedInputStream(zis));
 +
 +          Key key = new Key();
 +          Value val = new Value();
 +
 +          Mutation m = null;
 +          Text currentRow = null;
 +          int dirCount = 0;
 +
 +          while (true) {
 +            key.readFields(in);
 +            val.readFields(in);
 +
 +            Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow();
 +            Text metadataRow = new KeyExtent(new Text(tableInfo.tableId), endRow, null).getMetadataEntry();
 +
 +            Text cq;
 +
 +            if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +              String oldName = new Path(key.getColumnQualifier().toString()).getName();
 +              String newName = fileNameMappings.get(oldName);
 +
 +              if (newName == null) {
 +                throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +                    "File " + oldName + " does not exist in import dir");
 +              }
 +
 +              cq = new Text(bulkDir + "/" + newName);
 +            } else {
 +              cq = key.getColumnQualifier();
 +            }
 +
 +            if (m == null) {
 +              // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
-               String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++              String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
 +
 +              // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
 +              String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
 +
 +              m = new Mutation(metadataRow);
-               TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++              TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
 +              currentRow = metadataRow;
 +            }
 +
 +            if (!currentRow.equals(metadataRow)) {
 +              mbw.addMutation(m);
 +
 +              // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
-               String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++              String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
 +
 +              // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
 +              String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
 +
 +              m = new Mutation(metadataRow);
-               TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++              TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
 +            }
 +
 +            m.put(key.getColumnFamily(), cq, val);
 +
 +            if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
 +              mbw.addMutation(m);
 +              break; // its the last column in the last row
 +            }
 +          }
 +
 +          break;
 +        }
 +      }
 +
 +      return new MoveExportedFiles(tableInfo);
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error reading " + path + " " + ioe.getMessage());
 +    } finally {
 +      if (zis != null) {
 +        try {
 +          zis.close();
 +        } catch (IOException ioe) {
 +          log.warn("Failed to close zip file ", ioe);
 +        }
 +      }
 +
 +      if (mbw != null) {
 +        mbw.close();
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Given options for tables (across multiple volumes), construct an absolute path using the unique name within the chosen volume
 +   *
 +   * @return An absolute, unique path for the imported table
 +   */
 +  protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) {
 +    // We can try to spread out the tablet dirs across all volumes
 +    String tableDir = master.getFileSystem().choose(tableDirs);
 +
 +    // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
 +    return tableDir + "/" + tableInfo.tableId + "/" + tabletDir;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master environment) throws Exception {
 +    MetadataTableUtil.deleteTable(tableInfo.tableId, false, SystemCredentials.get(), environment.getMasterLock());
 +  }
 +}
 +
 +class MapImportFileNames extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  MapImportFileNames(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master environment) throws Exception {
 +
 +    Path path = new Path(tableInfo.importDir, "mappings.txt");
 +
 +    BufferedWriter mappingsWriter = null;
 +
 +    try {
 +      VolumeManager fs = environment.getFileSystem();
 +
 +      fs.mkdirs(new Path(tableInfo.importDir));
 +
 +      FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
 +
 +      UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
-       mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), Constants.UTF8));
++      mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), UTF_8));
 +
 +      for (FileStatus fileStatus : files) {
 +        String fileName = fileStatus.getPath().getName();
 +        log.info("filename " + fileStatus.getPath().toString());
 +        String sa[] = fileName.split("\\.");
 +        String extension = "";
 +        if (sa.length > 1) {
 +          extension = sa[sa.length - 1];
 +
 +          if (!FileOperations.getValidExtensions().contains(extension)) {
 +            continue;
 +          }
 +        } else {
 +          // assume it is a map file
 +          extension = Constants.MAPFILE_EXTENSION;
 +        }
 +
 +        String newName = "I" + namer.getNextName() + "." + extension;
 +
 +        mappingsWriter.append(fileName);
 +        mappingsWriter.append(':');
 +        mappingsWriter.append(newName);
 +        mappingsWriter.newLine();
 +      }
 +
 +      mappingsWriter.close();
 +      mappingsWriter = null;
 +
 +      return new PopulateMetadataTable(tableInfo);
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error writing mapping file " + path + " " + ioe.getMessage());
 +    } finally {
 +      if (mappingsWriter != null)
 +        try {
 +          mappingsWriter.close();
 +        } catch (IOException ioe) {
 +          log.warn("Failed to close " + path, ioe);
 +        }
 +    }
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir));
 +  }
 +}
 +
 +class CreateImportDir extends MasterRepo {
 +  private static final Logger log = Logger.getLogger(CreateImportDir.class);
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  CreateImportDir(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +
 +    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
 +
 +    Path exportDir = new Path(tableInfo.exportDir);
 +    String[] tableDirs = ServerConstants.getTablesDirs();
 +
 +    log.info("Looking for matching filesystem for " + exportDir + " from options " + Arrays.toString(tableDirs));
 +    Path base = master.getFileSystem().matchingFileSystem(exportDir, tableDirs);
 +    log.info("Chose base table directory of " + base);
 +    Path directory = new Path(base, tableInfo.tableId);
 +
 +    Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
 +
 +    tableInfo.importDir = newBulkDir.toString();
 +
 +    log.info("Using import dir: " + tableInfo.importDir);
 +
 +    return new MapImportFileNames(tableInfo);
 +  }
 +}
 +
 +class ImportPopulateZookeeper extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  ImportPopulateZookeeper(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT);
 +  }
 +
 +  private Map<String,String> getExportedProps(VolumeManager fs) throws Exception {
 +
 +    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
 +
 +    try {
 +      FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
 +      return TableOperationsImpl.getExportedProps(ns, path);
 +    } catch (IOException ioe) {
 +      throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Error reading table props from " + path + " " + ioe.getMessage());
 +    }
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    // reserve the table name in zookeeper or fail
 +
 +    Utils.tableNameLock.lock();
 +    try {
 +      // write tableName & tableId to zookeeper
 +      Instance instance = HdfsZooInstance.getInstance();
 +
 +      Utils.checkTableDoesNotExist(instance, tableInfo.tableName, tableInfo.tableId, TableOperation.CREATE);
 +
 +      String namespace = Tables.qualify(tableInfo.tableName).getFirst();
 +      String namespaceId = Namespaces.getNamespaceId(instance, namespace);
 +      TableManager.getInstance().addTable(tableInfo.tableId, namespaceId, tableInfo.tableName, NodeExistsPolicy.OVERWRITE);
 +
 +      Tables.clearCache(instance);
 +    } finally {
 +      Utils.tableNameLock.unlock();
 +    }
 +
 +    for (Entry<String,String> entry : getExportedProps(env.getFileSystem()).entrySet())
 +      if (!TablePropUtil.setTableProperty(tableInfo.tableId, entry.getKey(), entry.getValue())) {
 +        throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +            "Invalid table property " + entry.getKey());
 +      }
 +
 +    return new CreateImportDir(tableInfo);
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Instance instance = HdfsZooInstance.getInstance();
 +    TableManager.getInstance().removeTable(tableInfo.tableId);
 +    Utils.unreserveTable(tableInfo.tableId, tid, true);
 +    Tables.clearCache(instance);
 +  }
 +}
 +
 +class ImportSetupPermissions extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  public ImportSetupPermissions(ImportedTableInfo ti) {
 +    this.tableInfo = ti;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return 0;
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    // give all table permissions to the creator
 +    SecurityOperation security = AuditedSecurityOperation.getInstance();
 +    for (TablePermission permission : TablePermission.values()) {
 +      try {
 +        security.grantTablePermission(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.user, tableInfo.tableId, permission, tableInfo.namespaceId);
 +      } catch (ThriftSecurityException e) {
 +        Logger.getLogger(ImportSetupPermissions.class).error(e.getMessage(), e);
 +        throw e;
 +      }
 +    }
 +
 +    // setup permissions in zookeeper before table info in zookeeper
 +    // this way concurrent users will not get a spurious permission denied
 +    // error
 +    return new ImportPopulateZookeeper(tableInfo);
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    AuditedSecurityOperation.getInstance().deleteTable(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.tableId, tableInfo.namespaceId);
 +  }
 +}
 +
 +public class ImportTable extends MasterRepo {
 +  private static final long serialVersionUID = 1L;
 +
 +  private ImportedTableInfo tableInfo;
 +
 +  public ImportTable(String user, String tableName, String exportDir, String namespaceId) {
 +    tableInfo = new ImportedTableInfo();
 +    tableInfo.tableName = tableName;
 +    tableInfo.user = user;
 +    tableInfo.exportDir = exportDir;
 +    tableInfo.namespaceId = namespaceId;
 +  }
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid)
 +        + Utils.reserveNamespace(tableInfo.namespaceId, tid, false, true, TableOperation.IMPORT);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master env) throws Exception {
 +    checkVersions(env);
 +
 +    // first step is to reserve a table id.. if the machine fails during this step
 +    // it is ok to retry... the only side effect is that a table id may not be used
 +    // or skipped
 +
 +    // assuming only the master process is creating tables
 +
 +    Utils.idLock.lock();
 +    try {
 +      Instance instance = HdfsZooInstance.getInstance();
 +      tableInfo.tableId = Utils.getNextTableId(tableInfo.tableName, instance);
 +      return new ImportSetupPermissions(tableInfo);
 +    } finally {
 +      Utils.idLock.unlock();
 +    }
 +  }
 +
 +  public void checkVersions(Master env) throws ThriftTableOperationException {
 +    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
 +
 +    ZipInputStream zis = null;
 +
 +    try {
 +      zis = new ZipInputStream(env.getFileSystem().open(path));
 +
 +      Integer exportVersion = null;
 +      Integer dataVersion = null;
 +
 +      ZipEntry zipEntry;
 +      while ((zipEntry = zis.getNextEntry()) != null) {
 +        if (zipEntry.getName().equals(Constants.EXPORT_INFO_FILE)) {
-           BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++          BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
 +          String line = null;
 +          while ((line = in.readLine()) != null) {
 +            String sa[] = line.split(":", 2);
 +            if (sa[0].equals(ExportTable.EXPORT_VERSION_PROP)) {
 +              exportVersion = Integer.parseInt(sa[1]);
 +            } else if (sa[0].equals(ExportTable.DATA_VERSION_PROP)) {
 +              dataVersion = Integer.parseInt(sa[1]);
 +            }
 +          }
 +
 +          break;
 +        }
 +      }
 +
 +      zis.close();
 +      zis = null;
 +
 +      if (exportVersion == null || exportVersion > ExportTable.VERSION)
 +        throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +            "Incompatible export version " + exportVersion);
 +
 +      if (dataVersion == null || dataVersion > ServerConstants.DATA_VERSION)
 +        throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +            "Incompatible data version " + exportVersion);
 +
 +    } catch (IOException ioe) {
 +      log.warn(ioe.getMessage(), ioe);
 +      throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
 +          "Failed to read export metadata " + ioe.getMessage());
 +    } finally {
 +      if (zis != null)
 +        try {
 +          zis.close();
 +        } catch (IOException ioe) {
 +          log.warn(ioe.getMessage(), ioe);
 +        }
 +    }
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
 +    Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
 +  }
 +}


[03/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
index e44d8df,0000000..49b5d70
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
@@@ -1,496 -1,0 +1,496 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.File;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.Map;
 +import java.util.Set;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.commons.io.FileUtils;
 +import org.apache.commons.io.LineIterator;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +/**
 + * Tests that Accumulo is outputting audit messages as expected. Since this is using MiniAccumuloCluster, it could take a while if we test everything in
 + * isolation. We test blocks of related operations, run the whole test in one MiniAccumulo instance, trying to clean up objects between each test. The
 + * MiniAccumuloClusterTest sets up the log4j stuff differently to an installed instance, instead piping everything through stdout and writing to a set location
 + * so we have to find the logs and grep the bits we need out.
 + */
 +public class AuditMessageIT extends ConfigurableMacIT {
 +
 +  private static final String AUDIT_USER_1 = "AuditUser1";
 +  private static final String AUDIT_USER_2 = "AuditUser2";
 +  private static final String PASSWORD = "password";
 +  private static final String OLD_TEST_TABLE_NAME = "apples";
 +  private static final String NEW_TEST_TABLE_NAME = "oranges";
 +  private static final String THIRD_TEST_TABLE_NAME = "pears";
 +  private static final Authorizations auths = new Authorizations("private", "public");
 +
 +  @Override
 +  public int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Override
 +  public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {
 +    new File(cfg.getConfDir(), "auditLog.xml").delete();
 +  }
 +
 +  // Must be static to survive Junit re-initialising the class every time.
 +  private static String lastAuditTimestamp;
 +  private Connector auditConnector;
 +  private Connector conn;
 +
 +  private static ArrayList<String> findAuditMessage(ArrayList<String> input, String pattern) {
 +    ArrayList<String> result = new ArrayList<String>();
 +    for (String s : input) {
 +      if (s.matches(".*" + pattern + ".*"))
 +        result.add(s);
 +    }
 +    return result;
 +  }
 +
 +  /**
 +   * Returns a List of Audit messages that have been grep'd out of the MiniAccumuloCluster output.
 +   *
 +   * @param stepName
 +   *          A unique name for the test being executed, to identify the System.out messages.
 +   * @return A List of the Audit messages, sorted (so in chronological order).
 +   */
 +  private ArrayList<String> getAuditMessages(String stepName) throws IOException {
 +    // ACCUMULO-3144 Make sure we give the processes enough time to flush the write buffer
 +    try {
 +      Thread.sleep(2000);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IOException("Interrupted waiting for data to be flushed to output streams");
 +    }
 +
 +    for (MiniAccumuloClusterImpl.LogWriter lw : getCluster().getLogWriters()) {
 +      lw.flush();
 +    }
 +
 +    // Grab the audit messages
 +    System.out.println("Start of captured audit messages for step " + stepName);
 +
 +    ArrayList<String> result = new ArrayList<String>();
 +    for (File file : getCluster().getConfig().getLogDir().listFiles()) {
 +      // We want to grab the files called .out
 +      if (file.getName().contains(".out") && file.isFile() && file.canRead()) {
-         LineIterator it = FileUtils.lineIterator(file, Constants.UTF8.name());
++        LineIterator it = FileUtils.lineIterator(file, UTF_8.name());
 +        try {
 +          while (it.hasNext()) {
 +            String line = it.nextLine();
 +            if (line.matches(".* \\[" + AuditedSecurityOperation.AUDITLOG + "\\s*\\].*")) {
 +              // Only include the message if startTimestamp is null. or the message occurred after the startTimestamp value
 +              if ((lastAuditTimestamp == null) || (line.substring(0, 23).compareTo(lastAuditTimestamp) > 0))
 +                result.add(line);
 +            }
 +          }
 +        } finally {
 +          LineIterator.closeQuietly(it);
 +        }
 +      }
 +    }
 +    Collections.sort(result);
 +
 +    for (String s : result) {
 +      System.out.println(s);
 +    }
 +    System.out.println("End of captured audit messages for step " + stepName);
 +    if (result.size() > 0)
 +      lastAuditTimestamp = (result.get(result.size() - 1)).substring(0, 23);
 +
 +    return result;
 +  }
 +
 +  private void grantEverySystemPriv(Connector conn, String user) throws AccumuloSecurityException, AccumuloException {
 +    SystemPermission[] arrayOfP = new SystemPermission[] {SystemPermission.SYSTEM, SystemPermission.ALTER_TABLE, SystemPermission.ALTER_USER,
 +        SystemPermission.CREATE_TABLE, SystemPermission.CREATE_USER, SystemPermission.DROP_TABLE, SystemPermission.DROP_USER};
 +    for (SystemPermission p : arrayOfP) {
 +      conn.securityOperations().grantSystemPermission(user, p);
 +    }
 +  }
 +
 +  @Before
 +  public void resetInstance() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException {
 +    conn = getConnector();
 +
 +    // I don't want to recreate the instance for every test since it will take ages.
 +    // If we run every test as non-root users, I can drop these users every test which should effectively
 +    // reset the environment.
 +
 +    if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_1))
 +      conn.securityOperations().dropLocalUser(AUDIT_USER_1);
 +    if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
 +      conn.securityOperations().dropLocalUser(AUDIT_USER_2);
 +    if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
 +      conn.securityOperations().dropLocalUser(THIRD_TEST_TABLE_NAME);
 +    if (conn.tableOperations().exists(NEW_TEST_TABLE_NAME))
 +      conn.tableOperations().delete(NEW_TEST_TABLE_NAME);
 +    if (conn.tableOperations().exists(OLD_TEST_TABLE_NAME))
 +      conn.tableOperations().delete(OLD_TEST_TABLE_NAME);
 +
 +    // This will set the lastAuditTimestamp for the first test
 +    getAuditMessages("setup");
 +
 +  }
 +
 +  @Test
 +  public void testTableOperationsAudits() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_TABLE);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Testing activity begins here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
 +    Map<String,String> emptyMap = Collections.emptyMap();
 +    Set<String> emptySet = Collections.emptySet();
 +    auditConnector.tableOperations().clone(NEW_TEST_TABLE_NAME, OLD_TEST_TABLE_NAME, true, emptyMap, emptySet);
 +    auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().offline(NEW_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().delete(NEW_TEST_TABLE_NAME);
 +    // Testing activity ends here
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testTableOperationsAudits");
 +
 +    assertEquals(1, findAuditMessage(auditMessages, "action: createTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: renameTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: cloneTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: offlineTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +
 +  }
 +
 +  @Test
 +  public void testUserOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, InterruptedException, IOException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_USER);
 +    grantEverySystemPriv(conn, AUDIT_USER_1);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Start testing activities here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.securityOperations().createLocalUser(AUDIT_USER_2, new PasswordToken(PASSWORD));
 +
 +    // It seems only root can grant stuff.
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
 +    conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
 +    auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
 +    conn.securityOperations().grantTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
 +    conn.securityOperations().revokeTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
 +    auditConnector.securityOperations().changeLocalUserPassword(AUDIT_USER_2, new PasswordToken("anything"));
 +    auditConnector.securityOperations().changeUserAuthorizations(AUDIT_USER_2, auths);
 +    auditConnector.securityOperations().dropLocalUser(AUDIT_USER_2);
 +    // Stop testing activities here
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testUserOperationsAudits");
 +
 +    assertEquals(1, findAuditMessage(auditMessages, "action: createUser; targetUser: " + AUDIT_USER_2).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: grantSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: revokeSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: grantTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: revokeTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: changePassword; targetUser: " + AUDIT_USER_2 + "").size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: changeAuthorizations; targetUser: " + AUDIT_USER_2 + "; authorizations: " + auths.toString())
 +        .size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: dropUser; targetUser: " + AUDIT_USER_2).size());
 +  }
 +
 +  @Test
 +  public void testImportExportOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException,
 +      IOException, InterruptedException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
 +    grantEverySystemPriv(conn, AUDIT_USER_1);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Start testing activities here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
 +
 +    // Insert some play data
 +    BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
 +    Mutation m = new Mutation("myRow");
 +    m.put("cf1", "cq1", "v1");
 +    m.put("cf1", "cq2", "v3");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    // Prepare to export the table
 +    File exportDir = new File(getCluster().getConfig().getDir().toString() + "/export");
 +
 +    auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().exportTable(OLD_TEST_TABLE_NAME, exportDir.toString());
 +
 +    // We've exported the table metadata to the MiniAccumuloCluster root dir. Grab the .rf file path to re-import it
 +    File distCpTxt = new File(exportDir.toString() + "/distcp.txt");
 +    File importFile = null;
-     LineIterator it = FileUtils.lineIterator(distCpTxt, Constants.UTF8.name());
++    LineIterator it = FileUtils.lineIterator(distCpTxt, UTF_8.name());
 +
 +    // Just grab the first rf file, it will do for now.
 +    String filePrefix = "file:";
 +    try {
 +      while (it.hasNext() && importFile == null) {
 +        String line = it.nextLine();
 +        if (line.matches(".*\\.rf")) {
 +          importFile = new File(line.replaceFirst(filePrefix, ""));
 +        }
 +      }
 +    } finally {
 +      LineIterator.closeQuietly(it);
 +    }
 +    FileUtils.copyFileToDirectory(importFile, exportDir);
 +    auditConnector.tableOperations().importTable(NEW_TEST_TABLE_NAME, exportDir.toString());
 +
 +    // Now do a Directory (bulk) import of the same data.
 +    auditConnector.tableOperations().create(THIRD_TEST_TABLE_NAME);
 +    File failDir = new File(exportDir + "/tmp");
 +    failDir.mkdirs();
 +    auditConnector.tableOperations().importDirectory(THIRD_TEST_TABLE_NAME, exportDir.toString(), failDir.toString(), false);
 +    auditConnector.tableOperations().online(OLD_TEST_TABLE_NAME);
 +
 +    // Stop testing activities here
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testImportExportOperationsAudits");
 +
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME)).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_EXPORT_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, exportDir.toString())).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            String.format(AuditedSecurityOperation.CAN_IMPORT_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME, filePrefix + exportDir.toString())).size());
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME)).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(
 +            auditMessages,
 +            String.format(AuditedSecurityOperation.CAN_BULK_IMPORT_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME, filePrefix + exportDir.toString(), filePrefix
 +                + failDir.toString())).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "onlineTable", OLD_TEST_TABLE_NAME))
 +            .size());
 +
 +  }
 +
 +  @Test
 +  public void testDataOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
 +    grantEverySystemPriv(conn, AUDIT_USER_1);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Start testing activities here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
 +
 +    // Insert some play data
 +    BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
 +    Mutation m = new Mutation("myRow");
 +    m.put("cf1", "cq1", "v1");
 +    m.put("cf1", "cq2", "v3");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    // Start testing activities here
 +    // A regular scan
 +    Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
 +    for (Map.Entry<Key,Value> entry : scanner) {
 +      System.out.println("Scanner row: " + entry.getKey() + " " + entry.getValue());
 +    }
 +    scanner.close();
 +
 +    // A batch scan
 +    BatchScanner bs = auditConnector.createBatchScanner(OLD_TEST_TABLE_NAME, auths, 1);
 +    bs.fetchColumn(new Text("cf1"), new Text("cq1"));
 +    bs.setRanges(Arrays.asList(new Range("myRow", "myRow~")));
 +
 +    for (Map.Entry<Key,Value> entry : bs) {
 +      System.out.println("BatchScanner row: " + entry.getKey() + " " + entry.getValue());
 +    }
 +    bs.close();
 +
 +    // Delete some data.
 +    auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
 +
 +    // End of testing activities
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testDataOperationsAudits");
 +    assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
 +
 +  }
 +
 +  @Test
 +  public void testDeniedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    // Create our user with no privs
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.tableOperations().create(OLD_TEST_TABLE_NAME);
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +
 +    // Start testing activities
 +    // We should get denied or / failed audit messages here.
 +    // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
 +
 +    try {
 +      auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().clone(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME, true, Collections.<String,String> emptyMap(),
 +          Collections.<String> emptySet());
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
 +      scanner.iterator().next().getKey();
 +    } catch (RuntimeException ex) {}
 +    try {
 +      auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
 +    } catch (AccumuloSecurityException ex) {}
 +
 +    // ... that will do for now.
 +    // End of testing activities
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testDeniedAudits");
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_RENAME_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CLONE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(1, findAuditMessage(auditMessages, "operation: denied;.*" + "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
 +  }
 +
 +  @Test
 +  public void testFailedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    // Start testing activities
 +    // Test that we get a few "failed" audit messages come through when we tell it to do dumb stuff
 +    // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
 +    try {
 +      conn.securityOperations().dropLocalUser(AUDIT_USER_2);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      conn.securityOperations().createLocalUser("root", new PasswordToken("super secret"));
 +    } catch (AccumuloSecurityException ex) {}
 +    ArrayList<String> auditMessages = getAuditMessages("testFailedAudits");
 +    // ... that will do for now.
 +    // End of testing activities
 +
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.DROP_USER_AUDIT_TEMPLATE, AUDIT_USER_2)).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            String.format(AuditedSecurityOperation.REVOKE_SYSTEM_PERMISSION_AUDIT_TEMPLATE, SystemPermission.ALTER_TABLE, AUDIT_USER_2)).size());
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CREATE_USER_AUDIT_TEMPLATE, "root", "")).size());
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index c1e779e,0000000..8188deb
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@@ -1,1248 -1,0 +1,1248 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +import static org.junit.Assert.fail;
 +
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.EnumSet;
 +import java.util.Iterator;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.NamespaceExistsException;
 +import org.apache.accumulo.core.client.NamespaceNotEmptyException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.NamespaceOperations;
 +import org.apache.accumulo.core.client.admin.TableOperations;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.client.security.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.Filter;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.user.VersioningIterator;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
 +import org.apache.accumulo.test.functional.SimpleMacIT;
 +import org.apache.hadoop.io.Text;
 +import org.junit.After;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class NamespacesIT extends SimpleMacIT {
 +
 +  private Connector c;
 +  private String namespace;
 +
 +  @Override
 +  public int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Before
 +  public void setUpConnectorAndNamespace() throws Exception {
 +    // prepare a unique namespace and get a new root connector for each test
 +    c = getConnector();
 +    namespace = "ns_" + getUniqueNames(1)[0];
 +  }
 +
 +  @After
 +  public void swingMjölnir() throws Exception {
 +    // clean up any added tables, namespaces, and users, after each test
 +    for (String t : c.tableOperations().list())
 +      if (!Tables.qualify(t).getFirst().equals(Namespaces.ACCUMULO_NAMESPACE))
 +        c.tableOperations().delete(t);
 +    assertEquals(2, c.tableOperations().list().size());
 +    for (String n : c.namespaceOperations().list())
 +      if (!n.equals(Namespaces.ACCUMULO_NAMESPACE) && !n.equals(Namespaces.DEFAULT_NAMESPACE))
 +        c.namespaceOperations().delete(n);
 +    assertEquals(2, c.namespaceOperations().list().size());
 +    for (String u : c.securityOperations().listLocalUsers())
 +      if (!"root".equals(u))
 +        c.securityOperations().dropLocalUser(u);
 +    assertEquals(1, c.securityOperations().listLocalUsers().size());
 +  }
 +
 +  @Test
 +  public void checkReservedNamespaces() throws Exception {
 +    assertEquals(c.namespaceOperations().defaultNamespace(), Namespaces.DEFAULT_NAMESPACE);
 +    assertEquals(c.namespaceOperations().systemNamespace(), Namespaces.ACCUMULO_NAMESPACE);
 +  }
 +
 +  @Test
 +  public void checkBuiltInNamespaces() throws Exception {
 +    assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
 +    assertTrue(c.namespaceOperations().exists(Namespaces.ACCUMULO_NAMESPACE));
 +  }
 +
 +  @Test
 +  public void createTableInDefaultNamespace() throws Exception {
 +    String tableName = "1";
 +    c.tableOperations().create(tableName);
 +    assertTrue(c.tableOperations().exists(tableName));
 +  }
 +
 +  @Test(expected = AccumuloException.class)
 +  public void createTableInAccumuloNamespace() throws Exception {
 +    String tableName = Namespaces.ACCUMULO_NAMESPACE + ".1";
 +    assertFalse(c.tableOperations().exists(tableName));
 +    c.tableOperations().create(tableName); // should fail
 +  }
 +
 +  @Test(expected = AccumuloSecurityException.class)
 +  public void deleteDefaultNamespace() throws Exception {
 +    c.namespaceOperations().delete(Namespaces.DEFAULT_NAMESPACE); // should fail
 +  }
 +
 +  @Test(expected = AccumuloSecurityException.class)
 +  public void deleteAccumuloNamespace() throws Exception {
 +    c.namespaceOperations().delete(Namespaces.ACCUMULO_NAMESPACE); // should fail
 +  }
 +
 +  @Test
 +  public void createTableInMissingNamespace() throws Exception {
 +    String t = namespace + ".1";
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t));
 +    try {
 +      c.tableOperations().create(t);
 +      fail();
 +    } catch (AccumuloException e) {
 +      assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
 +      assertFalse(c.namespaceOperations().exists(namespace));
 +      assertFalse(c.tableOperations().exists(t));
 +    }
 +  }
 +
 +  @Test
 +  public void createAndDeleteNamespace() throws Exception {
 +    String t1 = namespace + ".1";
 +    String t2 = namespace + ".2";
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    try {
 +      c.namespaceOperations().delete(namespace);
 +    } catch (NamespaceNotFoundException e) {}
 +    try {
 +      c.tableOperations().delete(t1);
 +    } catch (TableNotFoundException e) {
 +      assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
 +    }
 +    c.namespaceOperations().create(namespace);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.tableOperations().create(t1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.tableOperations().create(t2);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t2));
 +    c.tableOperations().delete(t1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t2));
 +    c.tableOperations().delete(t2);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.namespaceOperations().delete(namespace);
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +  }
 +
 +  @Test(expected = NamespaceNotEmptyException.class)
 +  public void deleteNonEmptyNamespace() throws Exception {
 +    String tableName1 = namespace + ".1";
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(tableName1));
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(tableName1));
 +    c.namespaceOperations().delete(namespace); // should fail
 +  }
 +
 +  @Test
 +  public void verifyPropertyInheritance() throws Exception {
 +    String t0 = "0";
 +    String t1 = namespace + ".1";
 +    String t2 = namespace + ".2";
 +
 +    String k = Property.TABLE_SCAN_MAXMEM.getKey();
 +    String v = "42K";
 +
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    c.tableOperations().create(t0);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t0));
 +
 +    // verify no property
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(t1, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // set property and verify
 +    c.namespaceOperations().setProperty(namespace, k, v);
 +    assertTrue(checkNamespaceHasProp(namespace, k, v));
 +    assertTrue(checkTableHasProp(t1, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // add a new table to namespace and verify
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.tableOperations().create(t2);
 +    assertTrue(c.tableOperations().exists(t2));
 +    assertTrue(checkNamespaceHasProp(namespace, k, v));
 +    assertTrue(checkTableHasProp(t1, k, v));
 +    assertTrue(checkTableHasProp(t2, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // remove property and verify
 +    c.namespaceOperations().removeProperty(namespace, k);
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(t1, k, v));
 +    assertFalse(checkTableHasProp(t2, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // set property on default namespace and verify
 +    c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, k, v);
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(t1, k, v));
 +    assertFalse(checkTableHasProp(t2, k, v));
 +    assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertTrue(checkTableHasProp(t0, k, v));
 +
 +    // test that table properties override namespace properties
 +    String k2 = Property.TABLE_FILE_MAX.getKey();
 +    String v2 = "42";
 +    String table_v2 = "13";
 +
 +    // set new property on some
 +    c.namespaceOperations().setProperty(namespace, k2, v2);
 +    c.tableOperations().setProperty(t2, k2, table_v2);
 +    assertTrue(checkNamespaceHasProp(namespace, k2, v2));
 +    assertTrue(checkTableHasProp(t1, k2, v2));
 +    assertTrue(checkTableHasProp(t2, k2, table_v2));
 +
 +    c.tableOperations().delete(t1);
 +    c.tableOperations().delete(t2);
 +    c.tableOperations().delete(t0);
 +    c.namespaceOperations().delete(namespace);
 +  }
 +
 +  @Test
 +  public void verifyIteratorInheritance() throws Exception {
 +    String t1 = namespace + ".1";
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    String iterName = namespace + "_iter";
 +
 +    BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
 +    Mutation m = new Mutation("r");
 +    m.put("a", "b", new Value("abcde".getBytes()));
 +    bw.addMutation(m);
 +    bw.flush();
 +    bw.close();
 +
 +    IteratorSetting setting = new IteratorSetting(250, iterName, SimpleFilter.class.getName());
 +
 +    // verify can see inserted entry
 +    Scanner s = c.createScanner(t1, Authorizations.EMPTY);
 +    assertTrue(s.iterator().hasNext());
 +    assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
 +    assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
 +
 +    // verify entry is filtered out (also, verify conflict checking API)
 +    c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
 +    c.namespaceOperations().attachIterator(namespace, setting);
 +    UtilWaitThread.sleep(2*1000);
 +    try {
 +      c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
 +      fail();
 +    } catch (AccumuloException e) {
 +      assertEquals(IllegalArgumentException.class.getName(), e.getCause().getClass().getName());
 +    }
 +    IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan);
 +    assertEquals(setting, setting2);
 +    assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
 +    assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
 +    s = c.createScanner(t1, Authorizations.EMPTY);
 +    assertFalse(s.iterator().hasNext());
 +
 +    // verify can see inserted entry again
 +    c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
 +    UtilWaitThread.sleep(2*1000);
 +    assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
 +    assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
 +    s = c.createScanner(t1, Authorizations.EMPTY);
 +    assertTrue(s.iterator().hasNext());
 +  }
 +
 +  @Test
 +  public void cloneTable() throws Exception {
 +    String namespace2 = namespace + "_clone";
 +    String t1 = namespace + ".1";
 +    String t2 = namespace + ".2";
 +    String t3 = namespace2 + ".2";
 +    String k1 = Property.TABLE_FILE_MAX.getKey();
 +    String k2 = Property.TABLE_FILE_REPLICATION.getKey();
 +    String k1v1 = "55";
 +    String k1v2 = "66";
 +    String k2v1 = "5";
 +    String k2v2 = "6";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +
 +    try {
 +      // try to clone before namespace exists
 +      c.tableOperations().clone(t1, t3, false, null, null); // should fail
 +      fail();
 +    } catch (AccumuloException e) {
 +      assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
 +    }
 +
 +    // try to clone before when target tables exist
 +    c.namespaceOperations().create(namespace2);
 +    c.tableOperations().create(t2);
 +    c.tableOperations().create(t3);
 +    for (String t : Arrays.asList(t2, t3)) {
 +      try {
 +        c.tableOperations().clone(t1, t, false, null, null); // should fail
 +        fail();
 +      } catch (TableExistsException e) {
 +        c.tableOperations().delete(t);
 +      }
 +    }
 +
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +
 +    // set property with different values in two namespaces and a separate property with different values on the table and both namespaces
 +    assertFalse(checkNamespaceHasProp(namespace, k1, k1v1));
 +    assertFalse(checkNamespaceHasProp(namespace2, k1, k1v2));
 +    assertFalse(checkTableHasProp(t1, k1, k1v1));
 +    assertFalse(checkTableHasProp(t1, k1, k1v2));
 +    assertFalse(checkNamespaceHasProp(namespace, k2, k2v1));
 +    assertFalse(checkNamespaceHasProp(namespace2, k2, k2v1));
 +    assertFalse(checkTableHasProp(t1, k2, k2v1));
 +    assertFalse(checkTableHasProp(t1, k2, k2v2));
 +    c.namespaceOperations().setProperty(namespace, k1, k1v1);
 +    c.namespaceOperations().setProperty(namespace2, k1, k1v2);
 +    c.namespaceOperations().setProperty(namespace, k2, k2v1);
 +    c.namespaceOperations().setProperty(namespace2, k2, k2v1);
 +    c.tableOperations().setProperty(t1, k2, k2v2);
 +    assertTrue(checkNamespaceHasProp(namespace, k1, k1v1));
 +    assertTrue(checkNamespaceHasProp(namespace2, k1, k1v2));
 +    assertTrue(checkTableHasProp(t1, k1, k1v1));
 +    assertFalse(checkTableHasProp(t1, k1, k1v2));
 +    assertTrue(checkNamespaceHasProp(namespace, k2, k2v1));
 +    assertTrue(checkNamespaceHasProp(namespace2, k2, k2v1));
 +    assertFalse(checkTableHasProp(t1, k2, k2v1));
 +    assertTrue(checkTableHasProp(t1, k2, k2v2));
 +
 +    // clone twice, once in same namespace, once in another
 +    for (String t : Arrays.asList(t2, t3))
 +      c.tableOperations().clone(t1, t, false, null, null);
 +
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t2));
 +    assertTrue(c.tableOperations().exists(t3));
 +
 +    // verify the properties got transferred
 +    assertTrue(checkTableHasProp(t1, k1, k1v1));
 +    assertTrue(checkTableHasProp(t2, k1, k1v1));
 +    assertTrue(checkTableHasProp(t3, k1, k1v2));
 +    assertTrue(checkTableHasProp(t1, k2, k2v2));
 +    assertTrue(checkTableHasProp(t2, k2, k2v2));
 +    assertTrue(checkTableHasProp(t3, k2, k2v2));
 +  }
 +
 +  @Test
 +  public void renameNamespaceWithTable() throws Exception {
 +    String namespace2 = namespace + "_renamed";
 +    String t1 = namespace + ".t";
 +    String t2 = namespace2 + ".t";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t2));
 +
 +    String namespaceId = c.namespaceOperations().namespaceIdMap().get(namespace);
 +    String tableId = c.tableOperations().tableIdMap().get(t1);
 +
 +    c.namespaceOperations().rename(namespace, namespace2);
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertTrue(c.tableOperations().exists(t2));
 +
 +    // verify id's didn't change
 +    String namespaceId2 = c.namespaceOperations().namespaceIdMap().get(namespace2);
 +    String tableId2 = c.tableOperations().tableIdMap().get(t2);
 +
 +    assertEquals(namespaceId, namespaceId2);
 +    assertEquals(tableId, tableId2);
 +  }
 +
 +  @Test
 +  public void verifyConstraintInheritance() throws Exception {
 +    String t1 = namespace + ".1";
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1, false);
 +    String constraintClassName = NumericValueConstraint.class.getName();
 +
 +    assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
 +    assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
 +
 +    c.namespaceOperations().addConstraint(namespace, constraintClassName);
 +    assertTrue(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
 +    assertTrue(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
 +    int num = c.namespaceOperations().listConstraints(namespace).get(constraintClassName);
 +    assertEquals(num, (int) c.tableOperations().listConstraints(t1).get(constraintClassName));
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +
 +    Mutation m1 = new Mutation("r1");
 +    Mutation m2 = new Mutation("r2");
 +    Mutation m3 = new Mutation("r3");
-     m1.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
-     m2.put("e", "f", new Value("123".getBytes(Constants.UTF8)));
-     m3.put("c", "d", new Value("zyxwv".getBytes(Constants.UTF8)));
++    m1.put("a", "b", new Value("abcde".getBytes(UTF_8)));
++    m2.put("e", "f", new Value("123".getBytes(UTF_8)));
++    m3.put("c", "d", new Value("zyxwv".getBytes(UTF_8)));
 +    BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
 +    bw.addMutations(Arrays.asList(m1, m2, m3));
 +    try {
 +      bw.close();
 +      fail();
 +    } catch (MutationsRejectedException e) {
 +      assertEquals(1, e.getConstraintViolationSummaries().size());
 +      assertEquals(2, e.getConstraintViolationSummaries().get(0).getNumberOfViolatingMutations());
 +    }
 +    c.namespaceOperations().removeConstraint(namespace, num);
 +    assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
 +    assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +
 +    bw = c.createBatchWriter(t1, new BatchWriterConfig());
 +    bw.addMutations(Arrays.asList(m1, m2, m3));
 +    bw.close();
 +  }
 +
 +  @Test
 +  public void renameTable() throws Exception {
 +    String namespace2 = namespace + "_renamed";
 +    String t1 = namespace + ".1";
 +    String t2 = namespace2 + ".2";
 +    String t3 = namespace + ".3";
 +    String t4 = namespace + ".4";
 +    String t5 = "5";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.namespaceOperations().create(namespace2);
 +
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +    assertFalse(c.tableOperations().exists(t4));
 +    assertFalse(c.tableOperations().exists(t5));
 +
 +    c.tableOperations().create(t1);
 +
 +    try {
 +      c.tableOperations().rename(t1, t2);
 +      fail();
 +    } catch (AccumuloException e) {
 +      // this is expected, because we don't allow renames across namespaces
 +      assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
 +      assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
 +      assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
 +    }
 +
 +    try {
 +      c.tableOperations().rename(t1, t5);
 +      fail();
 +    } catch (AccumuloException e) {
 +      // this is expected, because we don't allow renames across namespaces
 +      assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
 +      assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
 +      assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
 +    }
 +
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +    assertFalse(c.tableOperations().exists(t4));
 +    assertFalse(c.tableOperations().exists(t5));
 +
 +    // fully qualified rename
 +    c.tableOperations().rename(t1, t3);
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertTrue(c.tableOperations().exists(t3));
 +    assertFalse(c.tableOperations().exists(t4));
 +    assertFalse(c.tableOperations().exists(t5));
 +  }
 +
 +  /**
 +   * Tests new Namespace permissions as well as modifications to Table permissions because of namespaces. Checks each permission to first make sure the user
 +   * doesn't have permission to perform the action, then root grants them the permission and we check to make sure they could perform the action.
 +   */
 +  @Test
 +  public void testPermissions() throws Exception {
 +    String u1 = "u1";
 +    String u2 = "u2";
 +    PasswordToken pass = new PasswordToken("pass");
 +
 +    String n1 = namespace;
 +    String t1 = n1 + ".1";
 +    String t2 = n1 + ".2";
 +    String t3 = n1 + ".3";
 +
 +    String n2 = namespace + "_2";
 +
 +    c.namespaceOperations().create(n1);
 +    c.tableOperations().create(t1);
 +
 +    c.securityOperations().createLocalUser(u1, pass);
 +
 +    Connector user1Con = c.getInstance().getConnector(u1, pass);
 +
 +    try {
 +      user1Con.tableOperations().create(t2);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
 +    user1Con.tableOperations().create(t2);
 +    assertTrue(c.tableOperations().list().contains(t2));
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
 +
 +    try {
 +      user1Con.tableOperations().delete(t1);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
 +    user1Con.tableOperations().delete(t1);
 +    assertTrue(!c.tableOperations().list().contains(t1));
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
 +
 +    c.tableOperations().create(t3);
 +    BatchWriter bw = c.createBatchWriter(t3, null);
 +    Mutation m = new Mutation("row");
 +    m.put("cf", "cq", "value");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    Iterator<Entry<Key,Value>> i = user1Con.createScanner(t3, new Authorizations()).iterator();
 +    try {
 +      i.next();
 +      fail();
 +    } catch (RuntimeException e) {
 +      assertEquals(AccumuloSecurityException.class.getName(), e.getCause().getClass().getName());
 +      expectPermissionDenied((AccumuloSecurityException) e.getCause());
 +    }
 +
 +    m = new Mutation(u1);
 +    m.put("cf", "cq", "turtles");
 +    bw = user1Con.createBatchWriter(t3, null);
 +    try {
 +      bw.addMutation(m);
 +      bw.close();
 +      fail();
 +    } catch (MutationsRejectedException e) {
 +      assertEquals(1, e.getAuthorizationFailuresMap().size());
 +      assertEquals(1, e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().size());
 +      switch (e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().iterator().next()) {
 +        case PERMISSION_DENIED:
 +          break;
 +        default:
 +          fail();
 +      }
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.READ);
 +    i = user1Con.createScanner(t3, new Authorizations()).iterator();
 +    assertTrue(i.hasNext());
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.READ);
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.WRITE);
 +    m = new Mutation(u1);
 +    m.put("cf", "cq", "turtles");
 +    bw = user1Con.createBatchWriter(t3, null);
 +    bw.addMutation(m);
 +    bw.close();
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.WRITE);
 +
 +    try {
 +      user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
 +    user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
 +    user1Con.tableOperations().removeProperty(t3, Property.TABLE_FILE_MAX.getKey());
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
 +
 +    try {
 +      user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "55");
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
 +    user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "42");
 +    user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
 +
 +    c.securityOperations().createLocalUser(u2, pass);
 +    try {
 +      user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.GRANT);
 +    user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
 +    user1Con.securityOperations().revokeNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.GRANT);
 +
 +    try {
 +      user1Con.namespaceOperations().create(n2);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
 +    user1Con.namespaceOperations().create(n2);
 +    c.securityOperations().revokeSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
 +
 +    c.securityOperations().revokeNamespacePermission(u1, n2, NamespacePermission.DROP_NAMESPACE);
 +    try {
 +      user1Con.namespaceOperations().delete(n2);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
 +    user1Con.namespaceOperations().delete(n2);
 +    c.securityOperations().revokeSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
 +
 +    try {
 +      user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
 +    user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
 +    user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
 +    c.securityOperations().revokeSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
 +  }
 +
 +  @Test
 +  public void verifySystemPropertyInheritance() throws Exception {
 +    String t1 = "1";
 +    String t2 = namespace + "." + t1;
 +    c.tableOperations().create(t1);
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t2);
 +
 +    // verify iterator inheritance
 +    _verifySystemPropertyInheritance(t1, t2, Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.sum", "20," + SimpleFilter.class.getName(), false);
 +
 +    // verify constraint inheritance
 +    _verifySystemPropertyInheritance(t1, t2, Property.TABLE_CONSTRAINT_PREFIX.getKey() + "42", NumericValueConstraint.class.getName(), false);
 +
 +    // verify other inheritance
 +    _verifySystemPropertyInheritance(t1, t2, Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "dummy", "dummy", true);
 +  }
 +
 +  private void _verifySystemPropertyInheritance(String defaultNamespaceTable, String namespaceTable, String k, String v, boolean systemNamespaceShouldInherit)
 +      throws Exception {
 +    // nobody should have any of these properties yet
 +    assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(RootTable.NAME, k, v));
 +    assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(namespaceTable, k, v));
 +
 +    // set the filter, verify that accumulo namespace is the only one unaffected
 +    c.instanceOperations().setProperty(k, v);
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +    assertTrue(c.instanceOperations().getSystemConfiguration().containsValue(v));
 +    assertEquals(systemNamespaceShouldInherit, checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
 +    assertEquals(systemNamespaceShouldInherit, checkTableHasProp(RootTable.NAME, k, v));
 +    assertEquals(systemNamespaceShouldInherit, checkTableHasProp(MetadataTable.NAME, k, v));
 +    assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertTrue(checkTableHasProp(defaultNamespaceTable, k, v));
 +    assertTrue(checkNamespaceHasProp(namespace, k, v));
 +    assertTrue(checkTableHasProp(namespaceTable, k, v));
 +
 +    // verify it is no longer inherited
 +    c.instanceOperations().removeProperty(k);
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +    assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(RootTable.NAME, k, v));
 +    assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(namespaceTable, k, v));
 +  }
 +
 +  @Test
 +  public void listNamespaces() throws Exception {
 +    SortedSet<String> namespaces = c.namespaceOperations().list();
 +    Map<String,String> map = c.namespaceOperations().namespaceIdMap();
 +    assertEquals(2, namespaces.size());
 +    assertEquals(2, map.size());
 +    assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
 +    assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
 +    assertFalse(namespaces.contains(namespace));
 +    assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
 +    assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
 +    assertNull(map.get(namespace));
 +
 +    c.namespaceOperations().create(namespace);
 +    namespaces = c.namespaceOperations().list();
 +    map = c.namespaceOperations().namespaceIdMap();
 +    assertEquals(3, namespaces.size());
 +    assertEquals(3, map.size());
 +    assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
 +    assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
 +    assertTrue(namespaces.contains(namespace));
 +    assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
 +    assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
 +    assertNotNull(map.get(namespace));
 +
 +    c.namespaceOperations().delete(namespace);
 +    namespaces = c.namespaceOperations().list();
 +    map = c.namespaceOperations().namespaceIdMap();
 +    assertEquals(2, namespaces.size());
 +    assertEquals(2, map.size());
 +    assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
 +    assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
 +    assertFalse(namespaces.contains(namespace));
 +    assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
 +    assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
 +    assertNull(map.get(namespace));
 +  }
 +
 +  @Test
 +  public void loadClass() throws Exception {
 +    assertTrue(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
 +    assertFalse(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, "dummy", SortedKeyValueIterator.class.getName()));
 +    try {
 +      c.namespaceOperations().testClassLoad(namespace, "dummy", "dummy");
 +      fail();
 +    } catch (NamespaceNotFoundException e) {}
 +  }
 +
 +  @Test
 +  public void testModifyingPermissions() throws Exception {
 +    String tableName = namespace + ".modify";
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName);
 +    assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
 +    c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
 +    assertFalse(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
 +    c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
 +    assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
 +    c.tableOperations().delete(tableName);
 +
 +    try {
 +      c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
 +    c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +    assertFalse(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
 +    c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +    assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
 +
 +    c.namespaceOperations().delete(namespace);
 +
 +    try {
 +      c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +  }
 +
 +  @Test
 +  public void verifyTableOperationsExceptions() throws Exception {
 +    String tableName = namespace + ".1";
 +    IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
 +    Text a = new Text("a");
 +    Text z = new Text("z");
 +    TableOperations ops = c.tableOperations();
 +
 +    // this one doesn't throw an exception, so don't fail; just check that it works
 +    assertFalse(ops.exists(tableName));
 +
 +    // table operations that should throw an AccumuloException caused by NamespaceNotFoundException
 +    int numRun = 0;
 +    ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.create(tableName);
 +            fail();
 +          case 1:
 +            ops.create("a");
 +            ops.clone("a", tableName, true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
 +            fail();
 +          case 2:
 +            ops.importTable(tableName, System.getProperty("user.dir") + "/target");
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(3, i); // check test integrity
 +            assertEquals(3, numRun); // check test integrity
 +            break ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof AccumuloException) || !(e.getCause() instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +
 +    // table operations that should throw an AccumuloException caused by a TableNotFoundException caused by a NamespaceNotFoundException
 +    // these are here because we didn't declare TableNotFoundException in the API :(
 +    numRun = 0;
 +    ACCUMULOEXCEPTIONS_TABLENOTFOUND: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.removeConstraint(tableName, 0);
 +            fail();
 +          case 1:
 +            ops.removeProperty(tableName, "a");
 +            fail();
 +          case 2:
 +            ops.setProperty(tableName, "a", "b");
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(3, i); // check test integrity
 +            assertEquals(3, numRun); // check test integrity
 +            break ACCUMULOEXCEPTIONS_TABLENOTFOUND;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof AccumuloException) || !(e.getCause() instanceof TableNotFoundException)
 +            || !(e.getCause().getCause() instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +
 +    // table operations that should throw a TableNotFoundException caused by NamespaceNotFoundException
 +    numRun = 0;
 +    TABLENOTFOUNDEXCEPTIONS: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.addConstraint(tableName, NumericValueConstraint.class.getName());
 +            fail();
 +          case 1:
 +            ops.addSplits(tableName, new TreeSet<Text>());
 +            fail();
 +          case 2:
 +            ops.attachIterator(tableName, setting);
 +            fail();
 +          case 3:
 +            ops.cancelCompaction(tableName);
 +            fail();
 +          case 4:
 +            ops.checkIteratorConflicts(tableName, setting, EnumSet.allOf(IteratorScope.class));
 +            fail();
 +          case 5:
 +            ops.clearLocatorCache(tableName);
 +            fail();
 +          case 6:
 +            ops.clone(tableName, "2", true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
 +            fail();
 +          case 7:
 +            ops.compact(tableName, a, z, true, true);
 +            fail();
 +          case 8:
 +            ops.delete(tableName);
 +            fail();
 +          case 9:
 +            ops.deleteRows(tableName, a, z);
 +            fail();
 +          case 10:
 +            ops.splitRangeByTablets(tableName, new Range(), 10);
 +            fail();
 +          case 11:
 +            ops.exportTable(tableName, namespace + "_dir");
 +            fail();
 +          case 12:
 +            ops.flush(tableName, a, z, true);
 +            fail();
 +          case 13:
 +            ops.getDiskUsage(Collections.singleton(tableName));
 +            fail();
 +          case 14:
 +            ops.getIteratorSetting(tableName, "a", IteratorScope.scan);
 +            fail();
 +          case 15:
 +            ops.getLocalityGroups(tableName);
 +            fail();
 +          case 16:
 +            ops.getMaxRow(tableName, Authorizations.EMPTY, a, true, z, true);
 +            fail();
 +          case 17:
 +            ops.getProperties(tableName);
 +            fail();
 +          case 18:
 +            ops.importDirectory(tableName, "", "", false);
 +            fail();
 +          case 19:
 +            ops.testClassLoad(tableName, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
 +            fail();
 +          case 20:
 +            ops.listConstraints(tableName);
 +            fail();
 +          case 21:
 +            ops.listIterators(tableName);
 +            fail();
 +          case 22:
 +            ops.listSplits(tableName);
 +            fail();
 +          case 23:
 +            ops.merge(tableName, a, z);
 +            fail();
 +          case 24:
 +            ops.offline(tableName, true);
 +            fail();
 +          case 25:
 +            ops.online(tableName, true);
 +            fail();
 +          case 26:
 +            ops.removeIterator(tableName, "a", EnumSet.of(IteratorScope.scan));
 +            fail();
 +          case 27:
 +            ops.rename(tableName, tableName + "2");
 +            fail();
 +          case 28:
 +            ops.setLocalityGroups(tableName, Collections.<String,Set<Text>> emptyMap());
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(29, i); // check test integrity
 +            assertEquals(29, numRun); // check test integrity
 +            break TABLENOTFOUNDEXCEPTIONS;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof TableNotFoundException) || !(e.getCause() instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +  }
 +
 +  @Test
 +  public void verifyNamespaceOperationsExceptions() throws Exception {
 +    IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
 +    NamespaceOperations ops = c.namespaceOperations();
 +
 +    // this one doesn't throw an exception, so don't fail; just check that it works
 +    assertFalse(ops.exists(namespace));
 +
 +    // namespace operations that should throw a NamespaceNotFoundException
 +    int numRun = 0;
 +    NAMESPACENOTFOUND: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.addConstraint(namespace, NumericValueConstraint.class.getName());
 +            fail();
 +          case 1:
 +            ops.attachIterator(namespace, setting);
 +            fail();
 +          case 2:
 +            ops.checkIteratorConflicts(namespace, setting, EnumSet.of(IteratorScope.scan));
 +            fail();
 +          case 3:
 +            ops.delete(namespace);
 +            fail();
 +          case 4:
 +            ops.getIteratorSetting(namespace, "thing", IteratorScope.scan);
 +            fail();
 +          case 5:
 +            ops.getProperties(namespace);
 +            fail();
 +          case 6:
 +            ops.listConstraints(namespace);
 +            fail();
 +          case 7:
 +            ops.listIterators(namespace);
 +            fail();
 +          case 8:
 +            ops.removeConstraint(namespace, 1);
 +            fail();
 +          case 9:
 +            ops.removeIterator(namespace, "thing", EnumSet.allOf(IteratorScope.class));
 +            fail();
 +          case 10:
 +            ops.removeProperty(namespace, "a");
 +            fail();
 +          case 11:
 +            ops.rename(namespace, namespace + "2");
 +            fail();
 +          case 12:
 +            ops.setProperty(namespace, "k", "v");
 +            fail();
 +          case 13:
 +            ops.testClassLoad(namespace, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(14, i); // check test integrity
 +            assertEquals(14, numRun); // check test integrity
 +            break NAMESPACENOTFOUND;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +
 +    // namespace operations that should throw a NamespaceExistsException
 +    numRun = 0;
 +    NAMESPACEEXISTS: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.create(namespace + "0");
 +            ops.create(namespace + "0"); // should fail here
 +            fail();
 +          case 1:
 +            ops.create(namespace + i + "_1");
 +            ops.create(namespace + i + "_2");
 +            ops.rename(namespace + i + "_1", namespace + i + "_2"); // should fail here
 +            fail();
 +          case 2:
 +            ops.create(Namespaces.DEFAULT_NAMESPACE);
 +            fail();
 +          case 3:
 +            ops.create(Namespaces.ACCUMULO_NAMESPACE);
 +            fail();
 +          case 4:
 +            ops.create(namespace + i + "_1");
 +            ops.rename(namespace + i + "_1", Namespaces.DEFAULT_NAMESPACE); // should fail here
 +            fail();
 +          case 5:
 +            ops.create(namespace + i + "_1");
 +            ops.rename(namespace + i + "_1", Namespaces.ACCUMULO_NAMESPACE); // should fail here
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(6, i); // check test integrity
 +            assertEquals(6, numRun); // check test integrity
 +            break NAMESPACEEXISTS;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof NamespaceExistsException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +  }
 +
 +  private boolean checkTableHasProp(String t, String propKey, String propVal) {
 +    return checkHasProperty(t, propKey, propVal, true);
 +  }
 +
 +  private boolean checkNamespaceHasProp(String n, String propKey, String propVal) {
 +    return checkHasProperty(n, propKey, propVal, false);
 +  }
 +
 +  private boolean checkHasProperty(String name, String propKey, String propVal, boolean nameIsTable) {
 +    try {
 +      Iterable<Entry<String,String>> iterable = nameIsTable ? c.tableOperations().getProperties(name) : c.namespaceOperations().getProperties(name);
 +      for (Entry<String,String> e : iterable)
 +        if (propKey.equals(e.getKey()))
 +          return propVal.equals(e.getValue());
 +      return false;
 +    } catch (Exception e) {
 +      fail();
 +      return false;
 +    }
 +  }
 +
 +  public static class SimpleFilter extends Filter {
 +    @Override
 +    public boolean accept(Key k, Value v) {
 +      if (k.getColumnFamily().toString().equals("a"))
 +        return false;
 +      return true;
 +    }
 +  }
 +
 +  private void expectPermissionDenied(AccumuloSecurityException sec) {
 +    assertEquals(sec.getSecurityErrorCode().getClass(), SecurityErrorCode.class);
 +    switch (sec.getSecurityErrorCode()) {
 +      case PERMISSION_DENIED:
 +        break;
 +      default:
 +        fail();
 +    }
 +  }
 +
 +}


[49/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Encoding.java
index 761fc1f,b26b363..259f783
--- a/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.util.Base64;
  import org.apache.hadoop.io.Text;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
index 7b84dc8,d1994ce..e103ac6
--- a/core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  public class FastFormat {
    // this 7 to 8 times faster than String.format("%s%06d",prefix, num)

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java
index 167933f,05067b5..c156311
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.core.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Iterator;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
index 90268ef,e71a377..4a0f9ef
--- a/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.Instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
index a57839b,617c079..66ad8f5
--- a/core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
@@@ -16,9 -16,10 +16,9 @@@
   */
  package org.apache.accumulo.core.util;
  
- import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
+ import java.nio.ByteBuffer;
 -
  import org.apache.accumulo.core.Constants;
  import org.apache.hadoop.io.Text;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index 8f0812c,c276056..a386c04
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.client;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertNotEquals;
  
@@@ -161,14 -155,8 +161,14 @@@ public class BatchWriterConfigTest 
      bwConfig.setMaxWriteThreads(24);
      bwConfig.setTimeout(3, TimeUnit.SECONDS);
      bytes = createBytes(bwConfig);
-     assertEquals("     v#maxWriteThreads=24,timeout=3000", new String(bytes, StandardCharsets.UTF_8));
+     assertEquals("     v#maxWriteThreads=24,timeout=3000", new String(bytes, UTF_8));
      checkBytes(bwConfig, bytes);
 +    
 +    // test human-readable durability
 +    bwConfig = new BatchWriterConfig();
 +    bwConfig.setDurability(Durability.LOG);
 +    bytes = createBytes(bwConfig);
-     assertEquals("     e#durability=LOG", new String(bytes, StandardCharsets.UTF_8));
++    assertEquals("     e#durability=LOG", new String(bytes, UTF_8));
    }
  
    @Test

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
index 98de0b5,dce742f..5d0b604
--- a/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.client;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.easymock.EasyMock.createMock;
  import static org.easymock.EasyMock.expect;
  import static org.easymock.EasyMock.replay;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index 1e86f73,e2661e2..b45054a
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -17,6 -17,7 +17,7 @@@
  
  package org.apache.accumulo.core.client.mock;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertTrue;
  import static org.junit.Assert.fail;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
index 46607ea,7bb3fa4..bbed50c
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
@@@ -16,11 -16,12 +16,11 @@@
   */
  package org.apache.accumulo.core.client.security.tokens;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
  import java.io.IOException;
  import java.net.URL;
- import java.nio.charset.StandardCharsets;
--
  import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
  import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
  import org.junit.Assert;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
index a3ce8d4,878b5e0..3751f02
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.client.security.tokens;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import javax.security.auth.DestroyFailedException;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
index 8672836,d9a47d7..c76275b
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertFalse;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 591baaa,908c391..4326227
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertFalse;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
index cd26743,8022c14..0aa7649
--- a/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
@@@ -17,6 -17,8 +17,8 @@@
   */
  package org.apache.accumulo.core.data;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
  import java.io.ByteArrayOutputStream;
  import java.io.DataInputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
index ec516d0,d8179df..2649f90
--- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.iterators.user;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.TreeMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index cf91cb0,2577e48..1a46004
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.apache.accumulo.core.security.ColumnVisibility.quote;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
index 5ecfdfb,a4e9f02..c5746bc
--- a/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertFalse;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index d8bff67,523279a..c9aa5d2
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.easymock.EasyMock.createMock;
  import static org.easymock.EasyMock.createNiceMock;
  import static org.easymock.EasyMock.expect;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index d80124c,9078ef9..0774849
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.security.crypto;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 8f46c89,eef06c3..e8879a5
--- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.core.util.format;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.junit.Assert.assertFalse;
  import static org.junit.Assert.assertTrue;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 8884cd9,af31cf2..60ff4b4
--- a/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.fate;
  
- import java.nio.charset.Charset;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.EnumSet;
@@@ -38,8 -38,6 +39,7 @@@ import org.apache.zookeeper.KeeperExcep
   * A utility to administer FATE operations
   */
  public class AdminUtil<T> {
-   private static final Charset UTF8 = Charset.forName("UTF-8");
 +  private static final Logger log = Logger.getLogger(AdminUtil.class);
    
    private boolean exitOnError = false;
    

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
index 5fc1858,34e0235..0dc156e
--- a/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.fate;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
  import java.io.ByteArrayOutputStream;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
index ea7f02c,b2e263a..d10fcd2
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.fate.zookeeper;
  
- import java.nio.charset.Charset;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Arrays;
  import java.util.Iterator;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
index 74c525e,4907bc6..ee53ddd
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.fate.zookeeper;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
  import java.io.ByteArrayOutputStream;
  import java.io.DataInputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
index 5fef905,f61c214..1391d98
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.fate.zookeeper;
  
- import java.nio.charset.Charset;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.List;
@@@ -33,8 -34,7 +34,7 @@@ import org.apache.zookeeper.ZooKeeper
  import org.apache.zookeeper.data.Stat;
  
  public class ZooLock implements Watcher {
-   private static final Charset UTF8 = Charset.forName("UTF-8");
 -  protected static final Logger log = Logger.getLogger(ZooLock.class);
 +  private static final Logger log = Logger.getLogger(ZooLock.class);
    
    public static final String LOCK_PREFIX = "zlock-";
    

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
index ea7f4b9,2e2acc3..f1ba428
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.fate.zookeeper;
  
- import java.nio.charset.Charset;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
  import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index b6890c6,ff35364..0059af7
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.fate.zookeeper;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.net.UnknownHostException;
- import java.nio.charset.Charset;
  import java.util.HashMap;
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index b591a68,0000000..29cf95d
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -1,522 -1,0 +1,523 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.DataInput;
 +import java.io.DataOutput;
 +import java.io.IOException;
 +import java.math.BigInteger;
- import java.nio.charset.StandardCharsets;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator;
 +import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase.TokenSource;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.mapreduce.InputSplit;
 +import org.apache.log4j.Level;
 +
 +/**
 + * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
 + */
 +public class RangeInputSplit extends InputSplit implements Writable {
 +  private Range range;
 +  private String[] locations;
 +  private String tableId, tableName, instanceName, zooKeepers, principal;
 +  private TokenSource tokenSource;
 +  private String tokenFile;
 +  private AuthenticationToken token;
 +  private Boolean offline, mockInstance, isolatedScan, localIterators;
 +  private Authorizations auths;
 +  private Set<Pair<Text,Text>> fetchedColumns;
 +  private List<IteratorSetting> iterators;
 +  private Level level;
 +
 +  public RangeInputSplit() {
 +    range = new Range();
 +    locations = new String[0];
 +    tableName = "";
 +    tableId = "";
 +  }
 +
 +  public RangeInputSplit(RangeInputSplit split) throws IOException {
 +    this.setRange(split.getRange());
 +    this.setLocations(split.getLocations());
 +    this.setTableName(split.getTableName());
 +    this.setTableId(split.getTableId());
 +  }
 +
 +  protected RangeInputSplit(String table, String tableId, Range range, String[] locations) {
 +    this.range = range;
 +    setLocations(locations);
 +    this.tableName = table;
 +    this.tableId = tableId;
 +  }
 +
 +  public Range getRange() {
 +    return range;
 +  }
 +
 +  private static byte[] extractBytes(ByteSequence seq, int numBytes) {
 +    byte[] bytes = new byte[numBytes + 1];
 +    bytes[0] = 0;
 +    for (int i = 0; i < numBytes; i++) {
 +      if (i >= seq.length())
 +        bytes[i + 1] = 0;
 +      else
 +        bytes[i + 1] = seq.byteAt(i);
 +    }
 +    return bytes;
 +  }
 +
 +  public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
 +    int maxDepth = Math.min(Math.max(end.length(), start.length()), position.length());
 +    BigInteger startBI = new BigInteger(extractBytes(start, maxDepth));
 +    BigInteger endBI = new BigInteger(extractBytes(end, maxDepth));
 +    BigInteger positionBI = new BigInteger(extractBytes(position, maxDepth));
 +    return (float) (positionBI.subtract(startBI).doubleValue() / endBI.subtract(startBI).doubleValue());
 +  }
 +
 +  public float getProgress(Key currentKey) {
 +    if (currentKey == null)
 +      return 0f;
 +    if (range.getStartKey() != null && range.getEndKey() != null) {
 +      if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
 +        // just look at the row progress
 +        return getProgress(range.getStartKey().getRowData(), range.getEndKey().getRowData(), currentKey.getRowData());
 +      } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM) != 0) {
 +        // just look at the column family progress
 +        return getProgress(range.getStartKey().getColumnFamilyData(), range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
 +      } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM_COLQUAL) != 0) {
 +        // just look at the column qualifier progress
 +        return getProgress(range.getStartKey().getColumnQualifierData(), range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
 +      }
 +    }
 +    // if we can't figure it out, then claim no progress
 +    return 0f;
 +  }
 +
 +  /**
 +   * This implementation of length is only an estimate, it does not provide exact values. Do not have your code rely on this return value.
 +   */
 +  @Override
 +  public long getLength() throws IOException {
 +    Text startRow = range.isInfiniteStartKey() ? new Text(new byte[] {Byte.MIN_VALUE}) : range.getStartKey().getRow();
 +    Text stopRow = range.isInfiniteStopKey() ? new Text(new byte[] {Byte.MAX_VALUE}) : range.getEndKey().getRow();
 +    int maxCommon = Math.min(7, Math.min(startRow.getLength(), stopRow.getLength()));
 +    long diff = 0;
 +
 +    byte[] start = startRow.getBytes();
 +    byte[] stop = stopRow.getBytes();
 +    for (int i = 0; i < maxCommon; ++i) {
 +      diff |= 0xff & (start[i] ^ stop[i]);
 +      diff <<= Byte.SIZE;
 +    }
 +
 +    if (startRow.getLength() != stopRow.getLength())
 +      diff |= 0xff;
 +
 +    return diff + 1;
 +  }
 +
 +  @Override
 +  public String[] getLocations() throws IOException {
 +    return Arrays.copyOf(locations, locations.length);
 +  }
 +
 +  @Override
 +  public void readFields(DataInput in) throws IOException {
 +    range.readFields(in);
 +    tableName = in.readUTF();
 +    tableId = in.readUTF();
 +    int numLocs = in.readInt();
 +    locations = new String[numLocs];
 +    for (int i = 0; i < numLocs; ++i)
 +      locations[i] = in.readUTF();
 +
 +    if (in.readBoolean()) {
 +      isolatedScan = in.readBoolean();
 +    }
 +
 +    if (in.readBoolean()) {
 +      offline = in.readBoolean();
 +    }
 +
 +    if (in.readBoolean()) {
 +      localIterators = in.readBoolean();
 +    }
 +
 +    if (in.readBoolean()) {
 +      mockInstance = in.readBoolean();
 +    }
 +
 +    if (in.readBoolean()) {
 +      int numColumns = in.readInt();
 +      List<String> columns = new ArrayList<String>(numColumns);
 +      for (int i = 0; i < numColumns; i++) {
 +        columns.add(in.readUTF());
 +      }
 +
 +      fetchedColumns = InputConfigurator.deserializeFetchedColumns(columns);
 +    }
 +
 +    if (in.readBoolean()) {
 +      String strAuths = in.readUTF();
-       auths = new Authorizations(strAuths.getBytes(StandardCharsets.UTF_8));
++      auths = new Authorizations(strAuths.getBytes(UTF_8));
 +    }
 +
 +    if (in.readBoolean()) {
 +      principal = in.readUTF();
 +    }
 +
 +    if (in.readBoolean()) {
 +      int ordinal = in.readInt();
 +      this.tokenSource = TokenSource.values()[ordinal];
 +
 +      switch (this.tokenSource) {
 +        case INLINE:
 +          String tokenClass = in.readUTF();
-           byte[] base64TokenBytes = in.readUTF().getBytes(StandardCharsets.UTF_8);
++          byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
 +          byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
 +
 +          this.token = AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes);
 +          break;
 +
 +        case FILE:
 +          this.tokenFile = in.readUTF();
 +
 +          break;
 +        default:
 +          throw new IOException("Cannot parse unknown TokenSource ordinal");
 +      }
 +    }
 +
 +    if (in.readBoolean()) {
 +      instanceName = in.readUTF();
 +    }
 +
 +    if (in.readBoolean()) {
 +      zooKeepers = in.readUTF();
 +    }
 +
 +    if (in.readBoolean()) {
 +      int numIterators = in.readInt();
 +      iterators = new ArrayList<IteratorSetting>(numIterators);
 +      for (int i = 0; i < numIterators; i++) {
 +        iterators.add(new IteratorSetting(in));
 +      }
 +    }
 +
 +    if (in.readBoolean()) {
 +      level = Level.toLevel(in.readInt());
 +    }
 +  }
 +
 +  @Override
 +  public void write(DataOutput out) throws IOException {
 +    range.write(out);
 +    out.writeUTF(tableName);
 +    out.writeUTF(tableId);
 +    out.writeInt(locations.length);
 +    for (int i = 0; i < locations.length; ++i)
 +      out.writeUTF(locations[i]);
 +
 +    out.writeBoolean(null != isolatedScan);
 +    if (null != isolatedScan) {
 +      out.writeBoolean(isolatedScan);
 +    }
 +
 +    out.writeBoolean(null != offline);
 +    if (null != offline) {
 +      out.writeBoolean(offline);
 +    }
 +
 +    out.writeBoolean(null != localIterators);
 +    if (null != localIterators) {
 +      out.writeBoolean(localIterators);
 +    }
 +
 +    out.writeBoolean(null != mockInstance);
 +    if (null != mockInstance) {
 +      out.writeBoolean(mockInstance);
 +    }
 +
 +    out.writeBoolean(null != fetchedColumns);
 +    if (null != fetchedColumns) {
 +      String[] cols = InputConfigurator.serializeColumns(fetchedColumns);
 +      out.writeInt(cols.length);
 +      for (String col : cols) {
 +        out.writeUTF(col);
 +      }
 +    }
 +
 +    out.writeBoolean(null != auths);
 +    if (null != auths) {
 +      out.writeUTF(auths.serialize());
 +    }
 +
 +    out.writeBoolean(null != principal);
 +    if (null != principal) {
 +      out.writeUTF(principal);
 +    }
 +
 +    out.writeBoolean(null != tokenSource);
 +    if (null != tokenSource) {
 +      out.writeInt(tokenSource.ordinal());
 +
 +      if (null != token && null != tokenFile) {
 +        throw new IOException("Cannot use both inline AuthenticationToken and file-based AuthenticationToken");
 +      } else if (null != token) {
 +        out.writeUTF(token.getClass().getCanonicalName());
 +        out.writeUTF(Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
 +      } else {
 +        out.writeUTF(tokenFile);
 +      }
 +    }
 +
 +    out.writeBoolean(null != instanceName);
 +    if (null != instanceName) {
 +      out.writeUTF(instanceName);
 +    }
 +
 +    out.writeBoolean(null != zooKeepers);
 +    if (null != zooKeepers) {
 +      out.writeUTF(zooKeepers);
 +    }
 +
 +    out.writeBoolean(null != iterators);
 +    if (null != iterators) {
 +      out.writeInt(iterators.size());
 +      for (IteratorSetting iterator : iterators) {
 +        iterator.write(out);
 +      }
 +    }
 +
 +    out.writeBoolean(null != level);
 +    if (null != level) {
 +      out.writeInt(level.toInt());
 +    }
 +  }
 +
 +  @Override
 +  public String toString() {
 +    StringBuilder sb = new StringBuilder(256);
 +    sb.append("Range: ").append(range);
 +    sb.append(" Locations: ").append(Arrays.asList(locations));
 +    sb.append(" Table: ").append(tableName);
 +    sb.append(" TableID: ").append(tableId);
 +    sb.append(" InstanceName: ").append(instanceName);
 +    sb.append(" zooKeepers: ").append(zooKeepers);
 +    sb.append(" principal: ").append(principal);
 +    sb.append(" tokenSource: ").append(tokenSource);
 +    sb.append(" authenticationToken: ").append(token);
 +    sb.append(" authenticationTokenFile: ").append(tokenFile);
 +    sb.append(" Authorizations: ").append(auths);
 +    sb.append(" offlineScan: ").append(offline);
 +    sb.append(" mockInstance: ").append(mockInstance);
 +    sb.append(" isolatedScan: ").append(isolatedScan);
 +    sb.append(" localIterators: ").append(localIterators);
 +    sb.append(" fetchColumns: ").append(fetchedColumns);
 +    sb.append(" iterators: ").append(iterators);
 +    sb.append(" logLevel: ").append(level);
 +    return sb.toString();
 +  }
 +
 +  /**
 +   * Use {@link #getTableName}
 +   */
 +  @Deprecated
 +  public String getTable() {
 +    return getTableName();
 +  }
 +
 +  public String getTableName() {
 +    return tableName;
 +  }
 +
 +  /**
 +   * Use {@link #setTableName}
 +   */
 +  @Deprecated
 +  public void setTable(String table) {
 +    setTableName(table);
 +  }
 +
 +  public void setTableName(String table) {
 +    this.tableName = table;
 +  }
 +
 +  public void setTableId(String tableId) {
 +    this.tableId = tableId;
 +  }
 +
 +  public String getTableId() {
 +    return tableId;
 +  }
 +
 +  public Instance getInstance() {
 +    if (null == instanceName) {
 +      return null;
 +    }
 +
 +    if (isMockInstance()) {
 +      return new MockInstance(getInstanceName());
 +    }
 +
 +    if (null == zooKeepers) {
 +      return null;
 +    }
 +
 +    return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(getInstanceName()).withZkHosts(getZooKeepers()));
 +  }
 +
 +  public String getInstanceName() {
 +    return instanceName;
 +  }
 +
 +  public void setInstanceName(String instanceName) {
 +    this.instanceName = instanceName;
 +  }
 +
 +  public String getZooKeepers() {
 +    return zooKeepers;
 +  }
 +
 +  public void setZooKeepers(String zooKeepers) {
 +    this.zooKeepers = zooKeepers;
 +  }
 +
 +  public String getPrincipal() {
 +    return principal;
 +  }
 +
 +  public void setPrincipal(String principal) {
 +    this.principal = principal;
 +  }
 +
 +  public AuthenticationToken getToken() {
 +    return token;
 +  }
 +
 +  public void setToken(AuthenticationToken token) {
 +    this.tokenSource = TokenSource.INLINE;
 +    this.token = token;
 +  }
 +
 +  public void setToken(String tokenFile) {
 +    this.tokenSource = TokenSource.FILE;
 +    this.tokenFile = tokenFile;
 +  }
 +
 +  public Boolean isOffline() {
 +    return offline;
 +  }
 +
 +  public void setOffline(Boolean offline) {
 +    this.offline = offline;
 +  }
 +
 +  public void setLocations(String[] locations) {
 +    this.locations = Arrays.copyOf(locations, locations.length);
 +  }
 +
 +  public Boolean isMockInstance() {
 +    return mockInstance;
 +  }
 +
 +  public void setMockInstance(Boolean mockInstance) {
 +    this.mockInstance = mockInstance;
 +  }
 +
 +  public Boolean isIsolatedScan() {
 +    return isolatedScan;
 +  }
 +
 +  public void setIsolatedScan(Boolean isolatedScan) {
 +    this.isolatedScan = isolatedScan;
 +  }
 +
 +  public Authorizations getAuths() {
 +    return auths;
 +  }
 +
 +  public void setAuths(Authorizations auths) {
 +    this.auths = auths;
 +  }
 +
 +  public void setRange(Range range) {
 +    this.range = range;
 +  }
 +
 +  public Boolean usesLocalIterators() {
 +    return localIterators;
 +  }
 +
 +  public void setUsesLocalIterators(Boolean localIterators) {
 +    this.localIterators = localIterators;
 +  }
 +
 +  public Set<Pair<Text,Text>> getFetchedColumns() {
 +    return fetchedColumns;
 +  }
 +
 +  public void setFetchedColumns(Collection<Pair<Text,Text>> fetchedColumns) {
 +    this.fetchedColumns = new HashSet<Pair<Text,Text>>();
 +    for (Pair<Text,Text> columns : fetchedColumns) {
 +      this.fetchedColumns.add(columns);
 +    }
 +  }
 +
 +  public void setFetchedColumns(Set<Pair<Text,Text>> fetchedColumns) {
 +    this.fetchedColumns = fetchedColumns;
 +  }
 +
 +  public List<IteratorSetting> getIterators() {
 +    return iterators;
 +  }
 +
 +  public void setIterators(List<IteratorSetting> iterators) {
 +    this.iterators = iterators;
 +  }
 +
 +  public Level getLogLevel() {
 +    return level;
 +  }
 +
 +  public void setLogLevel(Level level) {
 +    this.level = level;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index 2d9fc04,0000000..ae1d46f
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,405 -1,0 +1,404 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
 +
 +import java.io.IOException;
 +import java.net.URI;
 +import java.net.URISyntaxException;
- import java.nio.charset.StandardCharsets;
- 
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.util.StringUtils;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum ConnectorInfo {
 +    IS_CONFIGURED, PRINCIPAL, TOKEN,
 +  }
 +
 +  public static enum TokenSource {
 +    FILE, INLINE;
 +
 +    private String prefix;
 +
 +    private TokenSource() {
 +      prefix = name().toLowerCase() + ":";
 +    }
 +
 +    public String prefix() {
 +      return prefix;
 +    }
 +  }
 +
 +  /**
 +   * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum InstanceOpts {
 +    TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
 +  }
 +
 +  /**
 +   * Configuration keys for general configuration options.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum GeneralOpts {
 +    LOG_LEVEL,
 +    VISIBILITY_CACHE_SIZE
 +  }
 +
 +  /**
 +   * Provides a configuration key for a given feature enum, prefixed by the implementingClass
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param e
 +   *          the enum used to provide the unique part of the configuration key
 +   * @return the configuration key
 +   * @since 1.6.0
 +   */
 +  protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
 +    return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
 +  }
 +
 +  /**
 +  * Provides a configuration key for a given feature enum.
 +  * 
 +  * @param e
 +  *          the enum used to provide the unique part of the configuration key
 +  * @return the configuration key
 +  */
 +  protected static String enumToConfKey(Enum<?> e) {
 +	  return  e.getDeclaringClass().getSimpleName() + "." +  StringUtils.camelize(e.name().toLowerCase());
 +  }
 +
 +  /**
 +   * Sets the connector information needed to communicate with Accumulo in this job.
 +   * 
 +   * <p>
 +   * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
 +   * conversion to a string, and is not intended to be secure.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param principal
 +   *          a valid Accumulo user name
 +   * @param token
 +   *          the user's password
 +   * @since 1.6.0
 +   */
 +  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
 +      throws AccumuloSecurityException {
 +    if (isConnectorInfoSet(implementingClass, conf))
 +      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
 +
 +    checkArgument(principal != null, "principal is null");
 +    checkArgument(token != null, "token is null");
 +    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
 +        TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
 +  }
 +
 +  /**
 +   * Sets the connector information needed to communicate with Accumulo in this job.
 +   * 
 +   * <p>
 +   * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
 +   * Configuration. Token file created with "bin/accumulo create-token".
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param principal
 +   *          a valid Accumulo user name
 +   * @param tokenFile
 +   *          the path to the token file in DFS
 +   * @since 1.6.0
 +   */
 +  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
 +    if (isConnectorInfoSet(implementingClass, conf))
 +      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
 +
 +    checkArgument(principal != null, "principal is null");
 +    checkArgument(tokenFile != null, "tokenFile is null");
 +
 +    try {
 +      DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
 +    } catch (URISyntaxException e) {
 +      throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
 +    }
 +
 +    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
 +  }
 +
 +  /**
 +   * Determines if the connector info has already been set for this instance.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the connector info has already been set, false otherwise
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
 +  }
 +
 +  /**
 +   * Gets the user name from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the principal
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
 +  }
 +
 +  /**
 +   * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the principal's authentication token
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   * @see #setConnectorInfo(Class, Configuration, String, String)
 +   */
 +  public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
 +    String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
 +    if (token == null || token.isEmpty())
 +      return null;
 +    if (token.startsWith(TokenSource.INLINE.prefix())) {
 +      String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
 +      if (args.length == 2)
-         return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(StandardCharsets.UTF_8)));
++        return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(UTF_8)));
 +    } else if (token.startsWith(TokenSource.FILE.prefix())) {
 +      String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
 +      return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
 +    }
 +
 +    throw new IllegalStateException("Token was not properly serialized into the configuration");
 +  }
 +
 +  /**
 +   * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
 +   * 
 +   * @param conf
 +   *          the Hadoop context for the configured job
 +   * @return path to the token file as a String
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
 +    FSDataInputStream in = null;
 +    try {
 +      URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
 +      Path path = null;
 +      for (URI u : uris) {
 +        if (u.toString().equals(tokenFile)) {
 +          path = new Path(u);
 +        }
 +      }
 +      if (path == null) {
 +        throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
 +      }
 +      FileSystem fs = FileSystem.get(conf);
 +      in = fs.open(path);
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
 +    }
 +    try (java.util.Scanner fileScanner = new java.util.Scanner(in)) {
 +      while (fileScanner.hasNextLine()) {
 +        Credentials creds = Credentials.deserialize(fileScanner.nextLine());
 +        if (principal.equals(creds.getPrincipal())) {
 +          return creds.getToken();
 +        }
 +      }
 +      throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
 +    }
 +  }
 +
 +  /**
 +   * Configures a {@link ZooKeeperInstance} for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param clientConfig
 +   *          client configuration for specifying connection timeouts, SSL connection options, etc.
 +   * @since 1.6.0
 +   */
 +  public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
 +    String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
 +    if (!conf.get(key, "").isEmpty())
 +      throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
 +    conf.set(key, "ZooKeeperInstance");
 +    if (clientConfig != null) {
 +      conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
 +    }
 +  }
 +
 +  /**
 +   * Configures a {@link MockInstance} for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param instanceName
 +   *          the Accumulo instance name
 +   * @since 1.6.0
 +   */
 +  public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
 +    String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
 +    if (!conf.get(key, "").isEmpty())
 +      throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
 +    conf.set(key, "MockInstance");
 +
 +    checkArgument(instanceName != null, "instanceName is null");
 +    conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
 +  }
 +
 +  /**
 +   * Initializes an Accumulo {@link Instance} based on the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return an Accumulo instance
 +   * @since 1.6.0
 +   * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
 +   * @see #setMockInstance(Class, Configuration, String)
 +   */
 +  public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
 +    String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
 +    if ("MockInstance".equals(instanceType))
 +      return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
 +    else if ("ZooKeeperInstance".equals(instanceType)) {
 +      String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
 +      if (clientConfigString == null) {
 +        String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
 +        String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
 +        return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
 +      } else {
 +        return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
 +      }
 +    } else if (instanceType.isEmpty())
 +      throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
 +    else
 +      throw new IllegalStateException("Unrecognized instance type " + instanceType);
 +  }
 +
 +  /**
 +   * Sets the log level for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param level
 +   *          the logging level
 +   * @since 1.6.0
 +   */
 +  public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
 +    checkArgument(level != null, "level is null");
 +    Logger.getLogger(implementingClass).setLevel(level);
 +    conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
 +  }
 +
 +  /**
 +   * Gets the log level from this configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the log level
 +   * @since 1.6.0
 +   * @see #setLogLevel(Class, Configuration, Level)
 +   */
 +  public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
 +    return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
 +  }
 +
 +  /**
 +   * Sets the valid visibility count for this job.
 +   * 
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param visibilityCacheSize
 +   *          the LRU cache size
 +   */
 +  public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
 +    conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
 +  }
 +
 +  /**
 +   * Gets the valid visibility count for this job.
 +   * 
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the valid visibility count
 +   */
 +  public static int getVisibilityCacheSize(Configuration conf) {
 +    return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
 +  }
 +
 +}


[24/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
index e44d8df,0000000..49b5d70
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
@@@ -1,496 -1,0 +1,496 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.File;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.Map;
 +import java.util.Set;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.commons.io.FileUtils;
 +import org.apache.commons.io.LineIterator;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +/**
 + * Tests that Accumulo is outputting audit messages as expected. Since this is using MiniAccumuloCluster, it could take a while if we test everything in
 + * isolation. We test blocks of related operations, run the whole test in one MiniAccumulo instance, trying to clean up objects between each test. The
 + * MiniAccumuloClusterTest sets up the log4j stuff differently to an installed instance, instead piping everything through stdout and writing to a set location
 + * so we have to find the logs and grep the bits we need out.
 + */
 +public class AuditMessageIT extends ConfigurableMacIT {
 +
 +  private static final String AUDIT_USER_1 = "AuditUser1";
 +  private static final String AUDIT_USER_2 = "AuditUser2";
 +  private static final String PASSWORD = "password";
 +  private static final String OLD_TEST_TABLE_NAME = "apples";
 +  private static final String NEW_TEST_TABLE_NAME = "oranges";
 +  private static final String THIRD_TEST_TABLE_NAME = "pears";
 +  private static final Authorizations auths = new Authorizations("private", "public");
 +
 +  @Override
 +  public int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Override
 +  public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {
 +    new File(cfg.getConfDir(), "auditLog.xml").delete();
 +  }
 +
 +  // Must be static to survive Junit re-initialising the class every time.
 +  private static String lastAuditTimestamp;
 +  private Connector auditConnector;
 +  private Connector conn;
 +
 +  private static ArrayList<String> findAuditMessage(ArrayList<String> input, String pattern) {
 +    ArrayList<String> result = new ArrayList<String>();
 +    for (String s : input) {
 +      if (s.matches(".*" + pattern + ".*"))
 +        result.add(s);
 +    }
 +    return result;
 +  }
 +
 +  /**
 +   * Returns a List of Audit messages that have been grep'd out of the MiniAccumuloCluster output.
 +   *
 +   * @param stepName
 +   *          A unique name for the test being executed, to identify the System.out messages.
 +   * @return A List of the Audit messages, sorted (so in chronological order).
 +   */
 +  private ArrayList<String> getAuditMessages(String stepName) throws IOException {
 +    // ACCUMULO-3144 Make sure we give the processes enough time to flush the write buffer
 +    try {
 +      Thread.sleep(2000);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IOException("Interrupted waiting for data to be flushed to output streams");
 +    }
 +
 +    for (MiniAccumuloClusterImpl.LogWriter lw : getCluster().getLogWriters()) {
 +      lw.flush();
 +    }
 +
 +    // Grab the audit messages
 +    System.out.println("Start of captured audit messages for step " + stepName);
 +
 +    ArrayList<String> result = new ArrayList<String>();
 +    for (File file : getCluster().getConfig().getLogDir().listFiles()) {
 +      // We want to grab the files called .out
 +      if (file.getName().contains(".out") && file.isFile() && file.canRead()) {
-         LineIterator it = FileUtils.lineIterator(file, Constants.UTF8.name());
++        LineIterator it = FileUtils.lineIterator(file, UTF_8.name());
 +        try {
 +          while (it.hasNext()) {
 +            String line = it.nextLine();
 +            if (line.matches(".* \\[" + AuditedSecurityOperation.AUDITLOG + "\\s*\\].*")) {
 +              // Only include the message if startTimestamp is null. or the message occurred after the startTimestamp value
 +              if ((lastAuditTimestamp == null) || (line.substring(0, 23).compareTo(lastAuditTimestamp) > 0))
 +                result.add(line);
 +            }
 +          }
 +        } finally {
 +          LineIterator.closeQuietly(it);
 +        }
 +      }
 +    }
 +    Collections.sort(result);
 +
 +    for (String s : result) {
 +      System.out.println(s);
 +    }
 +    System.out.println("End of captured audit messages for step " + stepName);
 +    if (result.size() > 0)
 +      lastAuditTimestamp = (result.get(result.size() - 1)).substring(0, 23);
 +
 +    return result;
 +  }
 +
 +  private void grantEverySystemPriv(Connector conn, String user) throws AccumuloSecurityException, AccumuloException {
 +    SystemPermission[] arrayOfP = new SystemPermission[] {SystemPermission.SYSTEM, SystemPermission.ALTER_TABLE, SystemPermission.ALTER_USER,
 +        SystemPermission.CREATE_TABLE, SystemPermission.CREATE_USER, SystemPermission.DROP_TABLE, SystemPermission.DROP_USER};
 +    for (SystemPermission p : arrayOfP) {
 +      conn.securityOperations().grantSystemPermission(user, p);
 +    }
 +  }
 +
 +  @Before
 +  public void resetInstance() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException {
 +    conn = getConnector();
 +
 +    // I don't want to recreate the instance for every test since it will take ages.
 +    // If we run every test as non-root users, I can drop these users every test which should effectively
 +    // reset the environment.
 +
 +    if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_1))
 +      conn.securityOperations().dropLocalUser(AUDIT_USER_1);
 +    if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
 +      conn.securityOperations().dropLocalUser(AUDIT_USER_2);
 +    if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
 +      conn.securityOperations().dropLocalUser(THIRD_TEST_TABLE_NAME);
 +    if (conn.tableOperations().exists(NEW_TEST_TABLE_NAME))
 +      conn.tableOperations().delete(NEW_TEST_TABLE_NAME);
 +    if (conn.tableOperations().exists(OLD_TEST_TABLE_NAME))
 +      conn.tableOperations().delete(OLD_TEST_TABLE_NAME);
 +
 +    // This will set the lastAuditTimestamp for the first test
 +    getAuditMessages("setup");
 +
 +  }
 +
 +  @Test
 +  public void testTableOperationsAudits() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_TABLE);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Testing activity begins here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
 +    Map<String,String> emptyMap = Collections.emptyMap();
 +    Set<String> emptySet = Collections.emptySet();
 +    auditConnector.tableOperations().clone(NEW_TEST_TABLE_NAME, OLD_TEST_TABLE_NAME, true, emptyMap, emptySet);
 +    auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().offline(NEW_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().delete(NEW_TEST_TABLE_NAME);
 +    // Testing activity ends here
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testTableOperationsAudits");
 +
 +    assertEquals(1, findAuditMessage(auditMessages, "action: createTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: renameTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: cloneTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: offlineTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +
 +  }
 +
 +  @Test
 +  public void testUserOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, InterruptedException, IOException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_USER);
 +    grantEverySystemPriv(conn, AUDIT_USER_1);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Start testing activities here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.securityOperations().createLocalUser(AUDIT_USER_2, new PasswordToken(PASSWORD));
 +
 +    // It seems only root can grant stuff.
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
 +    conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
 +    auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
 +    conn.securityOperations().grantTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
 +    conn.securityOperations().revokeTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
 +    auditConnector.securityOperations().changeLocalUserPassword(AUDIT_USER_2, new PasswordToken("anything"));
 +    auditConnector.securityOperations().changeUserAuthorizations(AUDIT_USER_2, auths);
 +    auditConnector.securityOperations().dropLocalUser(AUDIT_USER_2);
 +    // Stop testing activities here
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testUserOperationsAudits");
 +
 +    assertEquals(1, findAuditMessage(auditMessages, "action: createUser; targetUser: " + AUDIT_USER_2).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: grantSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: revokeSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: grantTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "action: revokeTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: changePassword; targetUser: " + AUDIT_USER_2 + "").size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: changeAuthorizations; targetUser: " + AUDIT_USER_2 + "; authorizations: " + auths.toString())
 +        .size());
 +    assertEquals(1, findAuditMessage(auditMessages, "action: dropUser; targetUser: " + AUDIT_USER_2).size());
 +  }
 +
 +  @Test
 +  public void testImportExportOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException,
 +      IOException, InterruptedException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
 +    grantEverySystemPriv(conn, AUDIT_USER_1);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Start testing activities here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
 +
 +    // Insert some play data
 +    BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
 +    Mutation m = new Mutation("myRow");
 +    m.put("cf1", "cq1", "v1");
 +    m.put("cf1", "cq2", "v3");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    // Prepare to export the table
 +    File exportDir = new File(getCluster().getConfig().getDir().toString() + "/export");
 +
 +    auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
 +    auditConnector.tableOperations().exportTable(OLD_TEST_TABLE_NAME, exportDir.toString());
 +
 +    // We've exported the table metadata to the MiniAccumuloCluster root dir. Grab the .rf file path to re-import it
 +    File distCpTxt = new File(exportDir.toString() + "/distcp.txt");
 +    File importFile = null;
-     LineIterator it = FileUtils.lineIterator(distCpTxt, Constants.UTF8.name());
++    LineIterator it = FileUtils.lineIterator(distCpTxt, UTF_8.name());
 +
 +    // Just grab the first rf file, it will do for now.
 +    String filePrefix = "file:";
 +    try {
 +      while (it.hasNext() && importFile == null) {
 +        String line = it.nextLine();
 +        if (line.matches(".*\\.rf")) {
 +          importFile = new File(line.replaceFirst(filePrefix, ""));
 +        }
 +      }
 +    } finally {
 +      LineIterator.closeQuietly(it);
 +    }
 +    FileUtils.copyFileToDirectory(importFile, exportDir);
 +    auditConnector.tableOperations().importTable(NEW_TEST_TABLE_NAME, exportDir.toString());
 +
 +    // Now do a Directory (bulk) import of the same data.
 +    auditConnector.tableOperations().create(THIRD_TEST_TABLE_NAME);
 +    File failDir = new File(exportDir + "/tmp");
 +    failDir.mkdirs();
 +    auditConnector.tableOperations().importDirectory(THIRD_TEST_TABLE_NAME, exportDir.toString(), failDir.toString(), false);
 +    auditConnector.tableOperations().online(OLD_TEST_TABLE_NAME);
 +
 +    // Stop testing activities here
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testImportExportOperationsAudits");
 +
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME)).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_EXPORT_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, exportDir.toString())).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            String.format(AuditedSecurityOperation.CAN_IMPORT_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME, filePrefix + exportDir.toString())).size());
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME)).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(
 +            auditMessages,
 +            String.format(AuditedSecurityOperation.CAN_BULK_IMPORT_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME, filePrefix + exportDir.toString(), filePrefix
 +                + failDir.toString())).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "onlineTable", OLD_TEST_TABLE_NAME))
 +            .size());
 +
 +  }
 +
 +  @Test
 +  public void testDataOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
 +    conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
 +    grantEverySystemPriv(conn, AUDIT_USER_1);
 +
 +    // Connect as Audit User and do a bunch of stuff.
 +    // Start testing activities here
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +    auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
 +
 +    // Insert some play data
 +    BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
 +    Mutation m = new Mutation("myRow");
 +    m.put("cf1", "cq1", "v1");
 +    m.put("cf1", "cq2", "v3");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    // Start testing activities here
 +    // A regular scan
 +    Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
 +    for (Map.Entry<Key,Value> entry : scanner) {
 +      System.out.println("Scanner row: " + entry.getKey() + " " + entry.getValue());
 +    }
 +    scanner.close();
 +
 +    // A batch scan
 +    BatchScanner bs = auditConnector.createBatchScanner(OLD_TEST_TABLE_NAME, auths, 1);
 +    bs.fetchColumn(new Text("cf1"), new Text("cq1"));
 +    bs.setRanges(Arrays.asList(new Range("myRow", "myRow~")));
 +
 +    for (Map.Entry<Key,Value> entry : bs) {
 +      System.out.println("BatchScanner row: " + entry.getKey() + " " + entry.getValue());
 +    }
 +    bs.close();
 +
 +    // Delete some data.
 +    auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
 +
 +    // End of testing activities
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testDataOperationsAudits");
 +    assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
 +
 +  }
 +
 +  @Test
 +  public void testDeniedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    // Create our user with no privs
 +    conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
 +    conn.tableOperations().create(OLD_TEST_TABLE_NAME);
 +    auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
 +
 +    // Start testing activities
 +    // We should get denied or / failed audit messages here.
 +    // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
 +
 +    try {
 +      auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().clone(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME, true, Collections.<String,String> emptyMap(),
 +          Collections.<String> emptySet());
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
 +      scanner.iterator().next().getKey();
 +    } catch (RuntimeException ex) {}
 +    try {
 +      auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
 +    } catch (AccumuloSecurityException ex) {}
 +
 +    // ... that will do for now.
 +    // End of testing activities
 +
 +    ArrayList<String> auditMessages = getAuditMessages("testDeniedAudits");
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_RENAME_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CLONE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
 +    assertEquals(1,
 +        findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
 +            .size());
 +    assertEquals(1, findAuditMessage(auditMessages, "operation: denied;.*" + "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
 +  }
 +
 +  @Test
 +  public void testFailedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
 +      InterruptedException {
 +
 +    // Start testing activities
 +    // Test that we get a few "failed" audit messages come through when we tell it to do dumb stuff
 +    // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
 +    try {
 +      conn.securityOperations().dropLocalUser(AUDIT_USER_2);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
 +    } catch (AccumuloSecurityException ex) {}
 +    try {
 +      conn.securityOperations().createLocalUser("root", new PasswordToken("super secret"));
 +    } catch (AccumuloSecurityException ex) {}
 +    ArrayList<String> auditMessages = getAuditMessages("testFailedAudits");
 +    // ... that will do for now.
 +    // End of testing activities
 +
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.DROP_USER_AUDIT_TEMPLATE, AUDIT_USER_2)).size());
 +    assertEquals(
 +        1,
 +        findAuditMessage(auditMessages,
 +            String.format(AuditedSecurityOperation.REVOKE_SYSTEM_PERMISSION_AUDIT_TEMPLATE, SystemPermission.ALTER_TABLE, AUDIT_USER_2)).size());
 +    assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CREATE_USER_AUDIT_TEMPLATE, "root", "")).size());
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index c1e779e,0000000..8188deb
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@@ -1,1248 -1,0 +1,1248 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +import static org.junit.Assert.fail;
 +
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.EnumSet;
 +import java.util.Iterator;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.NamespaceExistsException;
 +import org.apache.accumulo.core.client.NamespaceNotEmptyException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.NamespaceOperations;
 +import org.apache.accumulo.core.client.admin.TableOperations;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.client.security.SecurityErrorCode;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.Filter;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.user.VersioningIterator;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.NamespacePermission;
 +import org.apache.accumulo.core.security.SystemPermission;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
 +import org.apache.accumulo.test.functional.SimpleMacIT;
 +import org.apache.hadoop.io.Text;
 +import org.junit.After;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class NamespacesIT extends SimpleMacIT {
 +
 +  private Connector c;
 +  private String namespace;
 +
 +  @Override
 +  public int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Before
 +  public void setUpConnectorAndNamespace() throws Exception {
 +    // prepare a unique namespace and get a new root connector for each test
 +    c = getConnector();
 +    namespace = "ns_" + getUniqueNames(1)[0];
 +  }
 +
 +  @After
 +  public void swingMjölnir() throws Exception {
 +    // clean up any added tables, namespaces, and users, after each test
 +    for (String t : c.tableOperations().list())
 +      if (!Tables.qualify(t).getFirst().equals(Namespaces.ACCUMULO_NAMESPACE))
 +        c.tableOperations().delete(t);
 +    assertEquals(2, c.tableOperations().list().size());
 +    for (String n : c.namespaceOperations().list())
 +      if (!n.equals(Namespaces.ACCUMULO_NAMESPACE) && !n.equals(Namespaces.DEFAULT_NAMESPACE))
 +        c.namespaceOperations().delete(n);
 +    assertEquals(2, c.namespaceOperations().list().size());
 +    for (String u : c.securityOperations().listLocalUsers())
 +      if (!"root".equals(u))
 +        c.securityOperations().dropLocalUser(u);
 +    assertEquals(1, c.securityOperations().listLocalUsers().size());
 +  }
 +
 +  @Test
 +  public void checkReservedNamespaces() throws Exception {
 +    assertEquals(c.namespaceOperations().defaultNamespace(), Namespaces.DEFAULT_NAMESPACE);
 +    assertEquals(c.namespaceOperations().systemNamespace(), Namespaces.ACCUMULO_NAMESPACE);
 +  }
 +
 +  @Test
 +  public void checkBuiltInNamespaces() throws Exception {
 +    assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
 +    assertTrue(c.namespaceOperations().exists(Namespaces.ACCUMULO_NAMESPACE));
 +  }
 +
 +  @Test
 +  public void createTableInDefaultNamespace() throws Exception {
 +    String tableName = "1";
 +    c.tableOperations().create(tableName);
 +    assertTrue(c.tableOperations().exists(tableName));
 +  }
 +
 +  @Test(expected = AccumuloException.class)
 +  public void createTableInAccumuloNamespace() throws Exception {
 +    String tableName = Namespaces.ACCUMULO_NAMESPACE + ".1";
 +    assertFalse(c.tableOperations().exists(tableName));
 +    c.tableOperations().create(tableName); // should fail
 +  }
 +
 +  @Test(expected = AccumuloSecurityException.class)
 +  public void deleteDefaultNamespace() throws Exception {
 +    c.namespaceOperations().delete(Namespaces.DEFAULT_NAMESPACE); // should fail
 +  }
 +
 +  @Test(expected = AccumuloSecurityException.class)
 +  public void deleteAccumuloNamespace() throws Exception {
 +    c.namespaceOperations().delete(Namespaces.ACCUMULO_NAMESPACE); // should fail
 +  }
 +
 +  @Test
 +  public void createTableInMissingNamespace() throws Exception {
 +    String t = namespace + ".1";
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t));
 +    try {
 +      c.tableOperations().create(t);
 +      fail();
 +    } catch (AccumuloException e) {
 +      assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
 +      assertFalse(c.namespaceOperations().exists(namespace));
 +      assertFalse(c.tableOperations().exists(t));
 +    }
 +  }
 +
 +  @Test
 +  public void createAndDeleteNamespace() throws Exception {
 +    String t1 = namespace + ".1";
 +    String t2 = namespace + ".2";
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    try {
 +      c.namespaceOperations().delete(namespace);
 +    } catch (NamespaceNotFoundException e) {}
 +    try {
 +      c.tableOperations().delete(t1);
 +    } catch (TableNotFoundException e) {
 +      assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
 +    }
 +    c.namespaceOperations().create(namespace);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.tableOperations().create(t1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.tableOperations().create(t2);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t2));
 +    c.tableOperations().delete(t1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t2));
 +    c.tableOperations().delete(t2);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.namespaceOperations().delete(namespace);
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +  }
 +
 +  @Test(expected = NamespaceNotEmptyException.class)
 +  public void deleteNonEmptyNamespace() throws Exception {
 +    String tableName1 = namespace + ".1";
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(tableName1));
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(tableName1));
 +    c.namespaceOperations().delete(namespace); // should fail
 +  }
 +
 +  @Test
 +  public void verifyPropertyInheritance() throws Exception {
 +    String t0 = "0";
 +    String t1 = namespace + ".1";
 +    String t2 = namespace + ".2";
 +
 +    String k = Property.TABLE_SCAN_MAXMEM.getKey();
 +    String v = "42K";
 +
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    c.tableOperations().create(t0);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t0));
 +
 +    // verify no property
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(t1, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // set property and verify
 +    c.namespaceOperations().setProperty(namespace, k, v);
 +    assertTrue(checkNamespaceHasProp(namespace, k, v));
 +    assertTrue(checkTableHasProp(t1, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // add a new table to namespace and verify
 +    assertFalse(c.tableOperations().exists(t2));
 +    c.tableOperations().create(t2);
 +    assertTrue(c.tableOperations().exists(t2));
 +    assertTrue(checkNamespaceHasProp(namespace, k, v));
 +    assertTrue(checkTableHasProp(t1, k, v));
 +    assertTrue(checkTableHasProp(t2, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // remove property and verify
 +    c.namespaceOperations().removeProperty(namespace, k);
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(t1, k, v));
 +    assertFalse(checkTableHasProp(t2, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(t0, k, v));
 +
 +    // set property on default namespace and verify
 +    c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, k, v);
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(t1, k, v));
 +    assertFalse(checkTableHasProp(t2, k, v));
 +    assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertTrue(checkTableHasProp(t0, k, v));
 +
 +    // test that table properties override namespace properties
 +    String k2 = Property.TABLE_FILE_MAX.getKey();
 +    String v2 = "42";
 +    String table_v2 = "13";
 +
 +    // set new property on some
 +    c.namespaceOperations().setProperty(namespace, k2, v2);
 +    c.tableOperations().setProperty(t2, k2, table_v2);
 +    assertTrue(checkNamespaceHasProp(namespace, k2, v2));
 +    assertTrue(checkTableHasProp(t1, k2, v2));
 +    assertTrue(checkTableHasProp(t2, k2, table_v2));
 +
 +    c.tableOperations().delete(t1);
 +    c.tableOperations().delete(t2);
 +    c.tableOperations().delete(t0);
 +    c.namespaceOperations().delete(namespace);
 +  }
 +
 +  @Test
 +  public void verifyIteratorInheritance() throws Exception {
 +    String t1 = namespace + ".1";
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    String iterName = namespace + "_iter";
 +
 +    BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
 +    Mutation m = new Mutation("r");
 +    m.put("a", "b", new Value("abcde".getBytes()));
 +    bw.addMutation(m);
 +    bw.flush();
 +    bw.close();
 +
 +    IteratorSetting setting = new IteratorSetting(250, iterName, SimpleFilter.class.getName());
 +
 +    // verify can see inserted entry
 +    Scanner s = c.createScanner(t1, Authorizations.EMPTY);
 +    assertTrue(s.iterator().hasNext());
 +    assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
 +    assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
 +
 +    // verify entry is filtered out (also, verify conflict checking API)
 +    c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
 +    c.namespaceOperations().attachIterator(namespace, setting);
 +    UtilWaitThread.sleep(2*1000);
 +    try {
 +      c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
 +      fail();
 +    } catch (AccumuloException e) {
 +      assertEquals(IllegalArgumentException.class.getName(), e.getCause().getClass().getName());
 +    }
 +    IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan);
 +    assertEquals(setting, setting2);
 +    assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
 +    assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
 +    s = c.createScanner(t1, Authorizations.EMPTY);
 +    assertFalse(s.iterator().hasNext());
 +
 +    // verify can see inserted entry again
 +    c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
 +    UtilWaitThread.sleep(2*1000);
 +    assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
 +    assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
 +    s = c.createScanner(t1, Authorizations.EMPTY);
 +    assertTrue(s.iterator().hasNext());
 +  }
 +
 +  @Test
 +  public void cloneTable() throws Exception {
 +    String namespace2 = namespace + "_clone";
 +    String t1 = namespace + ".1";
 +    String t2 = namespace + ".2";
 +    String t3 = namespace2 + ".2";
 +    String k1 = Property.TABLE_FILE_MAX.getKey();
 +    String k2 = Property.TABLE_FILE_REPLICATION.getKey();
 +    String k1v1 = "55";
 +    String k1v2 = "66";
 +    String k2v1 = "5";
 +    String k2v2 = "6";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +
 +    try {
 +      // try to clone before namespace exists
 +      c.tableOperations().clone(t1, t3, false, null, null); // should fail
 +      fail();
 +    } catch (AccumuloException e) {
 +      assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
 +    }
 +
 +    // try to clone before when target tables exist
 +    c.namespaceOperations().create(namespace2);
 +    c.tableOperations().create(t2);
 +    c.tableOperations().create(t3);
 +    for (String t : Arrays.asList(t2, t3)) {
 +      try {
 +        c.tableOperations().clone(t1, t, false, null, null); // should fail
 +        fail();
 +      } catch (TableExistsException e) {
 +        c.tableOperations().delete(t);
 +      }
 +    }
 +
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +
 +    // set property with different values in two namespaces and a separate property with different values on the table and both namespaces
 +    assertFalse(checkNamespaceHasProp(namespace, k1, k1v1));
 +    assertFalse(checkNamespaceHasProp(namespace2, k1, k1v2));
 +    assertFalse(checkTableHasProp(t1, k1, k1v1));
 +    assertFalse(checkTableHasProp(t1, k1, k1v2));
 +    assertFalse(checkNamespaceHasProp(namespace, k2, k2v1));
 +    assertFalse(checkNamespaceHasProp(namespace2, k2, k2v1));
 +    assertFalse(checkTableHasProp(t1, k2, k2v1));
 +    assertFalse(checkTableHasProp(t1, k2, k2v2));
 +    c.namespaceOperations().setProperty(namespace, k1, k1v1);
 +    c.namespaceOperations().setProperty(namespace2, k1, k1v2);
 +    c.namespaceOperations().setProperty(namespace, k2, k2v1);
 +    c.namespaceOperations().setProperty(namespace2, k2, k2v1);
 +    c.tableOperations().setProperty(t1, k2, k2v2);
 +    assertTrue(checkNamespaceHasProp(namespace, k1, k1v1));
 +    assertTrue(checkNamespaceHasProp(namespace2, k1, k1v2));
 +    assertTrue(checkTableHasProp(t1, k1, k1v1));
 +    assertFalse(checkTableHasProp(t1, k1, k1v2));
 +    assertTrue(checkNamespaceHasProp(namespace, k2, k2v1));
 +    assertTrue(checkNamespaceHasProp(namespace2, k2, k2v1));
 +    assertFalse(checkTableHasProp(t1, k2, k2v1));
 +    assertTrue(checkTableHasProp(t1, k2, k2v2));
 +
 +    // clone twice, once in same namespace, once in another
 +    for (String t : Arrays.asList(t2, t3))
 +      c.tableOperations().clone(t1, t, false, null, null);
 +
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertTrue(c.tableOperations().exists(t2));
 +    assertTrue(c.tableOperations().exists(t3));
 +
 +    // verify the properties got transferred
 +    assertTrue(checkTableHasProp(t1, k1, k1v1));
 +    assertTrue(checkTableHasProp(t2, k1, k1v1));
 +    assertTrue(checkTableHasProp(t3, k1, k1v2));
 +    assertTrue(checkTableHasProp(t1, k2, k2v2));
 +    assertTrue(checkTableHasProp(t2, k2, k2v2));
 +    assertTrue(checkTableHasProp(t3, k2, k2v2));
 +  }
 +
 +  @Test
 +  public void renameNamespaceWithTable() throws Exception {
 +    String namespace2 = namespace + "_renamed";
 +    String t1 = namespace + ".t";
 +    String t2 = namespace2 + ".t";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t2));
 +
 +    String namespaceId = c.namespaceOperations().namespaceIdMap().get(namespace);
 +    String tableId = c.tableOperations().tableIdMap().get(t1);
 +
 +    c.namespaceOperations().rename(namespace, namespace2);
 +    assertFalse(c.namespaceOperations().exists(namespace));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertTrue(c.tableOperations().exists(t2));
 +
 +    // verify id's didn't change
 +    String namespaceId2 = c.namespaceOperations().namespaceIdMap().get(namespace2);
 +    String tableId2 = c.tableOperations().tableIdMap().get(t2);
 +
 +    assertEquals(namespaceId, namespaceId2);
 +    assertEquals(tableId, tableId2);
 +  }
 +
 +  @Test
 +  public void verifyConstraintInheritance() throws Exception {
 +    String t1 = namespace + ".1";
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t1, false);
 +    String constraintClassName = NumericValueConstraint.class.getName();
 +
 +    assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
 +    assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
 +
 +    c.namespaceOperations().addConstraint(namespace, constraintClassName);
 +    assertTrue(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
 +    assertTrue(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
 +    int num = c.namespaceOperations().listConstraints(namespace).get(constraintClassName);
 +    assertEquals(num, (int) c.tableOperations().listConstraints(t1).get(constraintClassName));
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +
 +    Mutation m1 = new Mutation("r1");
 +    Mutation m2 = new Mutation("r2");
 +    Mutation m3 = new Mutation("r3");
-     m1.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
-     m2.put("e", "f", new Value("123".getBytes(Constants.UTF8)));
-     m3.put("c", "d", new Value("zyxwv".getBytes(Constants.UTF8)));
++    m1.put("a", "b", new Value("abcde".getBytes(UTF_8)));
++    m2.put("e", "f", new Value("123".getBytes(UTF_8)));
++    m3.put("c", "d", new Value("zyxwv".getBytes(UTF_8)));
 +    BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
 +    bw.addMutations(Arrays.asList(m1, m2, m3));
 +    try {
 +      bw.close();
 +      fail();
 +    } catch (MutationsRejectedException e) {
 +      assertEquals(1, e.getConstraintViolationSummaries().size());
 +      assertEquals(2, e.getConstraintViolationSummaries().get(0).getNumberOfViolatingMutations());
 +    }
 +    c.namespaceOperations().removeConstraint(namespace, num);
 +    assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
 +    assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +
 +    bw = c.createBatchWriter(t1, new BatchWriterConfig());
 +    bw.addMutations(Arrays.asList(m1, m2, m3));
 +    bw.close();
 +  }
 +
 +  @Test
 +  public void renameTable() throws Exception {
 +    String namespace2 = namespace + "_renamed";
 +    String t1 = namespace + ".1";
 +    String t2 = namespace2 + ".2";
 +    String t3 = namespace + ".3";
 +    String t4 = namespace + ".4";
 +    String t5 = "5";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.namespaceOperations().create(namespace2);
 +
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +    assertFalse(c.tableOperations().exists(t4));
 +    assertFalse(c.tableOperations().exists(t5));
 +
 +    c.tableOperations().create(t1);
 +
 +    try {
 +      c.tableOperations().rename(t1, t2);
 +      fail();
 +    } catch (AccumuloException e) {
 +      // this is expected, because we don't allow renames across namespaces
 +      assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
 +      assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
 +      assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
 +    }
 +
 +    try {
 +      c.tableOperations().rename(t1, t5);
 +      fail();
 +    } catch (AccumuloException e) {
 +      // this is expected, because we don't allow renames across namespaces
 +      assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
 +      assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
 +      assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
 +    }
 +
 +    assertTrue(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertFalse(c.tableOperations().exists(t3));
 +    assertFalse(c.tableOperations().exists(t4));
 +    assertFalse(c.tableOperations().exists(t5));
 +
 +    // fully qualified rename
 +    c.tableOperations().rename(t1, t3);
 +    assertFalse(c.tableOperations().exists(t1));
 +    assertFalse(c.tableOperations().exists(t2));
 +    assertTrue(c.tableOperations().exists(t3));
 +    assertFalse(c.tableOperations().exists(t4));
 +    assertFalse(c.tableOperations().exists(t5));
 +  }
 +
 +  /**
 +   * Tests new Namespace permissions as well as modifications to Table permissions because of namespaces. Checks each permission to first make sure the user
 +   * doesn't have permission to perform the action, then root grants them the permission and we check to make sure they could perform the action.
 +   */
 +  @Test
 +  public void testPermissions() throws Exception {
 +    String u1 = "u1";
 +    String u2 = "u2";
 +    PasswordToken pass = new PasswordToken("pass");
 +
 +    String n1 = namespace;
 +    String t1 = n1 + ".1";
 +    String t2 = n1 + ".2";
 +    String t3 = n1 + ".3";
 +
 +    String n2 = namespace + "_2";
 +
 +    c.namespaceOperations().create(n1);
 +    c.tableOperations().create(t1);
 +
 +    c.securityOperations().createLocalUser(u1, pass);
 +
 +    Connector user1Con = c.getInstance().getConnector(u1, pass);
 +
 +    try {
 +      user1Con.tableOperations().create(t2);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
 +    user1Con.tableOperations().create(t2);
 +    assertTrue(c.tableOperations().list().contains(t2));
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
 +
 +    try {
 +      user1Con.tableOperations().delete(t1);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
 +    user1Con.tableOperations().delete(t1);
 +    assertTrue(!c.tableOperations().list().contains(t1));
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
 +
 +    c.tableOperations().create(t3);
 +    BatchWriter bw = c.createBatchWriter(t3, null);
 +    Mutation m = new Mutation("row");
 +    m.put("cf", "cq", "value");
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    Iterator<Entry<Key,Value>> i = user1Con.createScanner(t3, new Authorizations()).iterator();
 +    try {
 +      i.next();
 +      fail();
 +    } catch (RuntimeException e) {
 +      assertEquals(AccumuloSecurityException.class.getName(), e.getCause().getClass().getName());
 +      expectPermissionDenied((AccumuloSecurityException) e.getCause());
 +    }
 +
 +    m = new Mutation(u1);
 +    m.put("cf", "cq", "turtles");
 +    bw = user1Con.createBatchWriter(t3, null);
 +    try {
 +      bw.addMutation(m);
 +      bw.close();
 +      fail();
 +    } catch (MutationsRejectedException e) {
 +      assertEquals(1, e.getAuthorizationFailuresMap().size());
 +      assertEquals(1, e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().size());
 +      switch (e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().iterator().next()) {
 +        case PERMISSION_DENIED:
 +          break;
 +        default:
 +          fail();
 +      }
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.READ);
 +    i = user1Con.createScanner(t3, new Authorizations()).iterator();
 +    assertTrue(i.hasNext());
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.READ);
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.WRITE);
 +    m = new Mutation(u1);
 +    m.put("cf", "cq", "turtles");
 +    bw = user1Con.createBatchWriter(t3, null);
 +    bw.addMutation(m);
 +    bw.close();
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.WRITE);
 +
 +    try {
 +      user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
 +    user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
 +    user1Con.tableOperations().removeProperty(t3, Property.TABLE_FILE_MAX.getKey());
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
 +
 +    try {
 +      user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "55");
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
 +    user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "42");
 +    user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
 +
 +    c.securityOperations().createLocalUser(u2, pass);
 +    try {
 +      user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.GRANT);
 +    user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
 +    user1Con.securityOperations().revokeNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
 +    c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.GRANT);
 +
 +    try {
 +      user1Con.namespaceOperations().create(n2);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
 +    user1Con.namespaceOperations().create(n2);
 +    c.securityOperations().revokeSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
 +
 +    c.securityOperations().revokeNamespacePermission(u1, n2, NamespacePermission.DROP_NAMESPACE);
 +    try {
 +      user1Con.namespaceOperations().delete(n2);
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
 +    user1Con.namespaceOperations().delete(n2);
 +    c.securityOperations().revokeSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
 +
 +    try {
 +      user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
 +      fail();
 +    } catch (AccumuloSecurityException e) {
 +      expectPermissionDenied(e);
 +    }
 +
 +    c.securityOperations().grantSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
 +    user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
 +    user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
 +    c.securityOperations().revokeSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
 +  }
 +
 +  @Test
 +  public void verifySystemPropertyInheritance() throws Exception {
 +    String t1 = "1";
 +    String t2 = namespace + "." + t1;
 +    c.tableOperations().create(t1);
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(t2);
 +
 +    // verify iterator inheritance
 +    _verifySystemPropertyInheritance(t1, t2, Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.sum", "20," + SimpleFilter.class.getName(), false);
 +
 +    // verify constraint inheritance
 +    _verifySystemPropertyInheritance(t1, t2, Property.TABLE_CONSTRAINT_PREFIX.getKey() + "42", NumericValueConstraint.class.getName(), false);
 +
 +    // verify other inheritance
 +    _verifySystemPropertyInheritance(t1, t2, Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "dummy", "dummy", true);
 +  }
 +
 +  private void _verifySystemPropertyInheritance(String defaultNamespaceTable, String namespaceTable, String k, String v, boolean systemNamespaceShouldInherit)
 +      throws Exception {
 +    // nobody should have any of these properties yet
 +    assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(RootTable.NAME, k, v));
 +    assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(namespaceTable, k, v));
 +
 +    // set the filter, verify that accumulo namespace is the only one unaffected
 +    c.instanceOperations().setProperty(k, v);
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +    assertTrue(c.instanceOperations().getSystemConfiguration().containsValue(v));
 +    assertEquals(systemNamespaceShouldInherit, checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
 +    assertEquals(systemNamespaceShouldInherit, checkTableHasProp(RootTable.NAME, k, v));
 +    assertEquals(systemNamespaceShouldInherit, checkTableHasProp(MetadataTable.NAME, k, v));
 +    assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertTrue(checkTableHasProp(defaultNamespaceTable, k, v));
 +    assertTrue(checkNamespaceHasProp(namespace, k, v));
 +    assertTrue(checkTableHasProp(namespaceTable, k, v));
 +
 +    // verify it is no longer inherited
 +    c.instanceOperations().removeProperty(k);
 +    // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
 +    UtilWaitThread.sleep(250);
 +    assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(RootTable.NAME, k, v));
 +    assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
 +    assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
 +    assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
 +    assertFalse(checkNamespaceHasProp(namespace, k, v));
 +    assertFalse(checkTableHasProp(namespaceTable, k, v));
 +  }
 +
 +  @Test
 +  public void listNamespaces() throws Exception {
 +    SortedSet<String> namespaces = c.namespaceOperations().list();
 +    Map<String,String> map = c.namespaceOperations().namespaceIdMap();
 +    assertEquals(2, namespaces.size());
 +    assertEquals(2, map.size());
 +    assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
 +    assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
 +    assertFalse(namespaces.contains(namespace));
 +    assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
 +    assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
 +    assertNull(map.get(namespace));
 +
 +    c.namespaceOperations().create(namespace);
 +    namespaces = c.namespaceOperations().list();
 +    map = c.namespaceOperations().namespaceIdMap();
 +    assertEquals(3, namespaces.size());
 +    assertEquals(3, map.size());
 +    assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
 +    assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
 +    assertTrue(namespaces.contains(namespace));
 +    assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
 +    assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
 +    assertNotNull(map.get(namespace));
 +
 +    c.namespaceOperations().delete(namespace);
 +    namespaces = c.namespaceOperations().list();
 +    map = c.namespaceOperations().namespaceIdMap();
 +    assertEquals(2, namespaces.size());
 +    assertEquals(2, map.size());
 +    assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
 +    assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
 +    assertFalse(namespaces.contains(namespace));
 +    assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
 +    assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
 +    assertNull(map.get(namespace));
 +  }
 +
 +  @Test
 +  public void loadClass() throws Exception {
 +    assertTrue(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
 +    assertFalse(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, "dummy", SortedKeyValueIterator.class.getName()));
 +    try {
 +      c.namespaceOperations().testClassLoad(namespace, "dummy", "dummy");
 +      fail();
 +    } catch (NamespaceNotFoundException e) {}
 +  }
 +
 +  @Test
 +  public void testModifyingPermissions() throws Exception {
 +    String tableName = namespace + ".modify";
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName);
 +    assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
 +    c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
 +    assertFalse(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
 +    c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
 +    assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
 +    c.tableOperations().delete(tableName);
 +
 +    try {
 +      c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
 +    c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +    assertFalse(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
 +    c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +    assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
 +
 +    c.namespaceOperations().delete(namespace);
 +
 +    try {
 +      c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +    try {
 +      c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
 +      fail();
 +    } catch (Exception e) {
 +      if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
 +        throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
 +    }
 +
 +  }
 +
 +  @Test
 +  public void verifyTableOperationsExceptions() throws Exception {
 +    String tableName = namespace + ".1";
 +    IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
 +    Text a = new Text("a");
 +    Text z = new Text("z");
 +    TableOperations ops = c.tableOperations();
 +
 +    // this one doesn't throw an exception, so don't fail; just check that it works
 +    assertFalse(ops.exists(tableName));
 +
 +    // table operations that should throw an AccumuloException caused by NamespaceNotFoundException
 +    int numRun = 0;
 +    ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.create(tableName);
 +            fail();
 +          case 1:
 +            ops.create("a");
 +            ops.clone("a", tableName, true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
 +            fail();
 +          case 2:
 +            ops.importTable(tableName, System.getProperty("user.dir") + "/target");
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(3, i); // check test integrity
 +            assertEquals(3, numRun); // check test integrity
 +            break ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof AccumuloException) || !(e.getCause() instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +
 +    // table operations that should throw an AccumuloException caused by a TableNotFoundException caused by a NamespaceNotFoundException
 +    // these are here because we didn't declare TableNotFoundException in the API :(
 +    numRun = 0;
 +    ACCUMULOEXCEPTIONS_TABLENOTFOUND: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.removeConstraint(tableName, 0);
 +            fail();
 +          case 1:
 +            ops.removeProperty(tableName, "a");
 +            fail();
 +          case 2:
 +            ops.setProperty(tableName, "a", "b");
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(3, i); // check test integrity
 +            assertEquals(3, numRun); // check test integrity
 +            break ACCUMULOEXCEPTIONS_TABLENOTFOUND;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof AccumuloException) || !(e.getCause() instanceof TableNotFoundException)
 +            || !(e.getCause().getCause() instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +
 +    // table operations that should throw a TableNotFoundException caused by NamespaceNotFoundException
 +    numRun = 0;
 +    TABLENOTFOUNDEXCEPTIONS: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.addConstraint(tableName, NumericValueConstraint.class.getName());
 +            fail();
 +          case 1:
 +            ops.addSplits(tableName, new TreeSet<Text>());
 +            fail();
 +          case 2:
 +            ops.attachIterator(tableName, setting);
 +            fail();
 +          case 3:
 +            ops.cancelCompaction(tableName);
 +            fail();
 +          case 4:
 +            ops.checkIteratorConflicts(tableName, setting, EnumSet.allOf(IteratorScope.class));
 +            fail();
 +          case 5:
 +            ops.clearLocatorCache(tableName);
 +            fail();
 +          case 6:
 +            ops.clone(tableName, "2", true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
 +            fail();
 +          case 7:
 +            ops.compact(tableName, a, z, true, true);
 +            fail();
 +          case 8:
 +            ops.delete(tableName);
 +            fail();
 +          case 9:
 +            ops.deleteRows(tableName, a, z);
 +            fail();
 +          case 10:
 +            ops.splitRangeByTablets(tableName, new Range(), 10);
 +            fail();
 +          case 11:
 +            ops.exportTable(tableName, namespace + "_dir");
 +            fail();
 +          case 12:
 +            ops.flush(tableName, a, z, true);
 +            fail();
 +          case 13:
 +            ops.getDiskUsage(Collections.singleton(tableName));
 +            fail();
 +          case 14:
 +            ops.getIteratorSetting(tableName, "a", IteratorScope.scan);
 +            fail();
 +          case 15:
 +            ops.getLocalityGroups(tableName);
 +            fail();
 +          case 16:
 +            ops.getMaxRow(tableName, Authorizations.EMPTY, a, true, z, true);
 +            fail();
 +          case 17:
 +            ops.getProperties(tableName);
 +            fail();
 +          case 18:
 +            ops.importDirectory(tableName, "", "", false);
 +            fail();
 +          case 19:
 +            ops.testClassLoad(tableName, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
 +            fail();
 +          case 20:
 +            ops.listConstraints(tableName);
 +            fail();
 +          case 21:
 +            ops.listIterators(tableName);
 +            fail();
 +          case 22:
 +            ops.listSplits(tableName);
 +            fail();
 +          case 23:
 +            ops.merge(tableName, a, z);
 +            fail();
 +          case 24:
 +            ops.offline(tableName, true);
 +            fail();
 +          case 25:
 +            ops.online(tableName, true);
 +            fail();
 +          case 26:
 +            ops.removeIterator(tableName, "a", EnumSet.of(IteratorScope.scan));
 +            fail();
 +          case 27:
 +            ops.rename(tableName, tableName + "2");
 +            fail();
 +          case 28:
 +            ops.setLocalityGroups(tableName, Collections.<String,Set<Text>> emptyMap());
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(29, i); // check test integrity
 +            assertEquals(29, numRun); // check test integrity
 +            break TABLENOTFOUNDEXCEPTIONS;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof TableNotFoundException) || !(e.getCause() instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +  }
 +
 +  @Test
 +  public void verifyNamespaceOperationsExceptions() throws Exception {
 +    IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
 +    NamespaceOperations ops = c.namespaceOperations();
 +
 +    // this one doesn't throw an exception, so don't fail; just check that it works
 +    assertFalse(ops.exists(namespace));
 +
 +    // namespace operations that should throw a NamespaceNotFoundException
 +    int numRun = 0;
 +    NAMESPACENOTFOUND: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.addConstraint(namespace, NumericValueConstraint.class.getName());
 +            fail();
 +          case 1:
 +            ops.attachIterator(namespace, setting);
 +            fail();
 +          case 2:
 +            ops.checkIteratorConflicts(namespace, setting, EnumSet.of(IteratorScope.scan));
 +            fail();
 +          case 3:
 +            ops.delete(namespace);
 +            fail();
 +          case 4:
 +            ops.getIteratorSetting(namespace, "thing", IteratorScope.scan);
 +            fail();
 +          case 5:
 +            ops.getProperties(namespace);
 +            fail();
 +          case 6:
 +            ops.listConstraints(namespace);
 +            fail();
 +          case 7:
 +            ops.listIterators(namespace);
 +            fail();
 +          case 8:
 +            ops.removeConstraint(namespace, 1);
 +            fail();
 +          case 9:
 +            ops.removeIterator(namespace, "thing", EnumSet.allOf(IteratorScope.class));
 +            fail();
 +          case 10:
 +            ops.removeProperty(namespace, "a");
 +            fail();
 +          case 11:
 +            ops.rename(namespace, namespace + "2");
 +            fail();
 +          case 12:
 +            ops.setProperty(namespace, "k", "v");
 +            fail();
 +          case 13:
 +            ops.testClassLoad(namespace, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(14, i); // check test integrity
 +            assertEquals(14, numRun); // check test integrity
 +            break NAMESPACENOTFOUND;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof NamespaceNotFoundException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +
 +    // namespace operations that should throw a NamespaceExistsException
 +    numRun = 0;
 +    NAMESPACEEXISTS: for (int i = 0;; ++i)
 +      try {
 +        switch (i) {
 +          case 0:
 +            ops.create(namespace + "0");
 +            ops.create(namespace + "0"); // should fail here
 +            fail();
 +          case 1:
 +            ops.create(namespace + i + "_1");
 +            ops.create(namespace + i + "_2");
 +            ops.rename(namespace + i + "_1", namespace + i + "_2"); // should fail here
 +            fail();
 +          case 2:
 +            ops.create(Namespaces.DEFAULT_NAMESPACE);
 +            fail();
 +          case 3:
 +            ops.create(Namespaces.ACCUMULO_NAMESPACE);
 +            fail();
 +          case 4:
 +            ops.create(namespace + i + "_1");
 +            ops.rename(namespace + i + "_1", Namespaces.DEFAULT_NAMESPACE); // should fail here
 +            fail();
 +          case 5:
 +            ops.create(namespace + i + "_1");
 +            ops.rename(namespace + i + "_1", Namespaces.ACCUMULO_NAMESPACE); // should fail here
 +            fail();
 +          default:
 +            // break out of infinite loop
 +            assertEquals(6, i); // check test integrity
 +            assertEquals(6, numRun); // check test integrity
 +            break NAMESPACEEXISTS;
 +        }
 +      } catch (Exception e) {
 +        numRun++;
 +        if (!(e instanceof NamespaceExistsException))
 +          throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
 +      }
 +  }
 +
 +  private boolean checkTableHasProp(String t, String propKey, String propVal) {
 +    return checkHasProperty(t, propKey, propVal, true);
 +  }
 +
 +  private boolean checkNamespaceHasProp(String n, String propKey, String propVal) {
 +    return checkHasProperty(n, propKey, propVal, false);
 +  }
 +
 +  private boolean checkHasProperty(String name, String propKey, String propVal, boolean nameIsTable) {
 +    try {
 +      Iterable<Entry<String,String>> iterable = nameIsTable ? c.tableOperations().getProperties(name) : c.namespaceOperations().getProperties(name);
 +      for (Entry<String,String> e : iterable)
 +        if (propKey.equals(e.getKey()))
 +          return propVal.equals(e.getValue());
 +      return false;
 +    } catch (Exception e) {
 +      fail();
 +      return false;
 +    }
 +  }
 +
 +  public static class SimpleFilter extends Filter {
 +    @Override
 +    public boolean accept(Key k, Value v) {
 +      if (k.getColumnFamily().toString().equals("a"))
 +        return false;
 +      return true;
 +    }
 +  }
 +
 +  private void expectPermissionDenied(AccumuloSecurityException sec) {
 +    assertEquals(sec.getSecurityErrorCode().getClass(), SecurityErrorCode.class);
 +    switch (sec.getSecurityErrorCode()) {
 +      case PERMISSION_DENIED:
 +        break;
 +      default:
 +        fail();
 +    }
 +  }
 +
 +}


[40/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index cf1f0ea,0000000..da83386
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,407 -1,0 +1,409 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.net.URI;
 +import java.net.URISyntaxException;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.util.StringUtils;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum ConnectorInfo {
 +    IS_CONFIGURED, PRINCIPAL, TOKEN,
 +  }
 +
 +  public static enum TokenSource {
 +    FILE, INLINE;
 +
 +    private String prefix;
 +
 +    private TokenSource() {
 +      prefix = name().toLowerCase() + ":";
 +    }
 +
 +    public String prefix() {
 +      return prefix;
 +    }
 +  }
 +
 +  /**
 +   * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum InstanceOpts {
 +    TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
 +  }
 +
 +  /**
 +   * Configuration keys for general configuration options.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum GeneralOpts {
 +    LOG_LEVEL,
 +    VISIBILITY_CACHE_SIZE
 +  }
 +
 +  /**
 +   * Provides a configuration key for a given feature enum, prefixed by the implementingClass
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param e
 +   *          the enum used to provide the unique part of the configuration key
 +   * @return the configuration key
 +   * @since 1.6.0
 +   */
 +  protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
 +    return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
 +  }
 +
 +  /**
 +  * Provides a configuration key for a given feature enum.
 +  * 
 +  * @param e
 +  *          the enum used to provide the unique part of the configuration key
 +  * @return the configuration key
 +  */
 +  protected static String enumToConfKey(Enum<?> e) {
 +	  return  e.getDeclaringClass().getSimpleName() + "." +  StringUtils.camelize(e.name().toLowerCase());
 +  }
 +
 +  /**
 +   * Sets the connector information needed to communicate with Accumulo in this job.
 +   * 
 +   * <p>
 +   * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
 +   * conversion to a string, and is not intended to be secure.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param principal
 +   *          a valid Accumulo user name
 +   * @param token
 +   *          the user's password
 +   * @since 1.6.0
 +   */
 +  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
 +      throws AccumuloSecurityException {
 +    if (isConnectorInfoSet(implementingClass, conf))
 +      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
 +
 +    ArgumentChecker.notNull(principal, token);
 +    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
 +        TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
 +  }
 +
 +  /**
 +   * Sets the connector information needed to communicate with Accumulo in this job.
 +   * 
 +   * <p>
 +   * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
 +   * Configuration. Token file created with "bin/accumulo create-token".
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param principal
 +   *          a valid Accumulo user name
 +   * @param tokenFile
 +   *          the path to the token file in DFS
 +   * @since 1.6.0
 +   */
 +  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
 +    if (isConnectorInfoSet(implementingClass, conf))
 +      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
 +
 +    ArgumentChecker.notNull(principal, tokenFile);
 +
 +    try {
 +      DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
 +    } catch (URISyntaxException e) {
 +      throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
 +    }
 +
 +    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
 +    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
 +  }
 +
 +  /**
 +   * Determines if the connector info has already been set for this instance.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the connector info has already been set, false otherwise
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
 +  }
 +
 +  /**
 +   * Gets the user name from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the principal
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
 +  }
 +
 +  /**
 +   * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the principal's authentication token
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   * @see #setConnectorInfo(Class, Configuration, String, String)
 +   */
 +  public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
 +    String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
 +    if (token == null || token.isEmpty())
 +      return null;
 +    if (token.startsWith(TokenSource.INLINE.prefix())) {
 +      String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
 +      if (args.length == 2)
-         return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(Constants.UTF8)));
++        return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(UTF_8)));
 +    } else if (token.startsWith(TokenSource.FILE.prefix())) {
 +      String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
 +      return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
 +    }
 +
 +    throw new IllegalStateException("Token was not properly serialized into the configuration");
 +  }
 +
 +  /**
 +   * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
 +   * 
 +   * @param conf
 +   *          the Hadoop context for the configured job
 +   * @return path to the token file as a String
 +   * @since 1.6.0
 +   * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
 +   */
 +  public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
 +    FSDataInputStream in = null;
 +    try {
 +      URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
 +      Path path = null;
 +      for (URI u : uris) {
 +        if (u.toString().equals(tokenFile)) {
 +          path = new Path(u);
 +        }
 +      }
 +      if (path == null) {
 +        throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
 +      }
 +      FileSystem fs = FileSystem.get(conf);
 +      in = fs.open(path);
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
 +    }
 +    java.util.Scanner fileScanner = new java.util.Scanner(in);
 +    try {
 +      while (fileScanner.hasNextLine()) {
 +        Credentials creds = Credentials.deserialize(fileScanner.nextLine());
 +        if (principal.equals(creds.getPrincipal())) {
 +          return creds.getToken();
 +        }
 +      }
 +      throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
 +    } finally {
 +      if (fileScanner != null && fileScanner.ioException() == null)
 +        fileScanner.close();
 +      else if (fileScanner.ioException() != null)
 +        throw new RuntimeException(fileScanner.ioException());
 +    }
 +  }
 +
 +  /**
 +   * Configures a {@link ZooKeeperInstance} for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param clientConfig
 +   *          client configuration for specifying connection timeouts, SSL connection options, etc.
 +   * @since 1.6.0
 +   */
 +  public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
 +    String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
 +    if (!conf.get(key, "").isEmpty())
 +      throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
 +    conf.set(key, "ZooKeeperInstance");
 +    if (clientConfig != null) {
 +      conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
 +    }
 +  }
 +
 +  /**
 +   * Configures a {@link MockInstance} for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param instanceName
 +   *          the Accumulo instance name
 +   * @since 1.6.0
 +   */
 +  public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
 +    String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
 +    if (!conf.get(key, "").isEmpty())
 +      throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
 +    conf.set(key, "MockInstance");
 +
 +    ArgumentChecker.notNull(instanceName);
 +    conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
 +  }
 +
 +  /**
 +   * Initializes an Accumulo {@link Instance} based on the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return an Accumulo instance
 +   * @since 1.6.0
 +   * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
 +   * @see #setMockInstance(Class, Configuration, String)
 +   */
 +  public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
 +    String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
 +    if ("MockInstance".equals(instanceType))
 +      return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
 +    else if ("ZooKeeperInstance".equals(instanceType)) {
 +      String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
 +      if (clientConfigString == null) {
 +        String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
 +        String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
 +        return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
 +      } else {
 +        return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
 +      }
 +    } else if (instanceType.isEmpty())
 +      throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
 +    else
 +      throw new IllegalStateException("Unrecognized instance type " + instanceType);
 +  }
 +
 +  /**
 +   * Sets the log level for this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param level
 +   *          the logging level
 +   * @since 1.6.0
 +   */
 +  public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
 +    ArgumentChecker.notNull(level);
 +    Logger.getLogger(implementingClass).setLevel(level);
 +    conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
 +  }
 +
 +  /**
 +   * Gets the log level from this configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the log level
 +   * @since 1.6.0
 +   * @see #setLogLevel(Class, Configuration, Level)
 +   */
 +  public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
 +    return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
 +  }
 +
 +  /**
 +   * Sets the valid visibility count for this job.
 +   * 
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param visibilityCacheSize
 +   *          the LRU cache size
 +   */
 +  public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
 +    conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
 +  }
 +
 +  /**
 +   * Gets the valid visibility count for this job.
 +   * 
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the valid visibility count
 +   */
 +  public static int getVisibilityCacheSize(Configuration conf) {
 +    return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
index 1ed23e3,0000000..f8a5d03
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@@ -1,795 -1,0 +1,795 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.core.util.ArgumentChecker.notNull;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.StringTokenizer;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientSideIteratorScanner;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.TabletLocator;
 +import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
 +import org.apache.accumulo.core.client.mock.impl.MockTabletLocator;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.MapWritable;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.util.StringUtils;
 +
 +import com.google.common.collect.Maps;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class InputConfigurator extends ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link Scanner}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum ScanOpts {
 +    TABLE_NAME, AUTHORIZATIONS, RANGES, COLUMNS, ITERATORS, TABLE_CONFIGS
 +  }
 +
 +  /**
 +   * Configuration keys for various features.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum Features {
 +    AUTO_ADJUST_RANGES, SCAN_ISOLATION, USE_LOCAL_ITERATORS, SCAN_OFFLINE
 +  }
 +
 +  /**
 +   * Sets the name of the input table, over which this job will scan.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table to use when the tablename is null in the write call
 +   * @since 1.6.0
 +   */
 +  public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
 +    notNull(tableName);
 +    conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
 +  }
 +
 +  /**
 +   * Sets the name of the input table, over which this job will scan.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @since 1.6.0
 +   */
 +  public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
 +  }
 +
 +  /**
 +   * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param auths
 +   *          the user's authorizations
 +   * @since 1.6.0
 +   */
 +  public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf, Authorizations auths) {
 +    if (auths != null && !auths.isEmpty())
 +      conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
 +  }
 +
 +  /**
 +   * Gets the authorizations to set for the scans from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the Accumulo scan authorizations
 +   * @since 1.6.0
 +   * @see #setScanAuthorizations(Class, Configuration, Authorizations)
 +   */
 +  public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
 +    String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
-     return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(Constants.UTF8));
++    return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(UTF_8));
 +  }
 +
 +  /**
 +   * Sets the input ranges to scan on all input tables for this job. If not set, the entire table will be scanned.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param ranges
 +   *          the ranges that will be mapped over
 +   * @throws IllegalArgumentException
 +   *           if the ranges cannot be encoded into base 64
 +   * @since 1.6.0
 +   */
 +  public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
 +    notNull(ranges);
 +
 +    ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
 +    try {
 +      for (Range r : ranges) {
 +        ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +        r.write(new DataOutputStream(baos));
 +        rangeStrings.add(Base64.encodeBase64String(baos.toByteArray()));
 +      }
 +      conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES), rangeStrings.toArray(new String[0]));
 +    } catch (IOException ex) {
 +      throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
 +    }
 +  }
 +
 +  /**
 +   * Gets the ranges to scan over from a job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the ranges
 +   * @throws IOException
 +   *           if the ranges have been encoded improperly
 +   * @since 1.6.0
 +   * @see #setRanges(Class, Configuration, Collection)
 +   */
 +  public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
 +
 +    Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
 +    List<Range> ranges = new ArrayList<Range>();
 +    for (String rangeString : encodedRanges) {
 +      ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes()));
 +      Range range = new Range();
 +      range.readFields(new DataInputStream(bais));
 +      ranges.add(range);
 +    }
 +    return ranges;
 +  }
 +
 +  /**
 +   * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return a list of iterators
 +   * @since 1.6.0
 +   * @see #addIterator(Class, Configuration, IteratorSetting)
 +   */
 +  public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
 +    String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
 +
 +    // If no iterators are present, return an empty list
 +    if (iterators == null || iterators.isEmpty())
 +      return new ArrayList<IteratorSetting>();
 +
 +    // Compose the set of iterators encoded in the job configuration
 +    StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
 +    List<IteratorSetting> list = new ArrayList<IteratorSetting>();
 +    try {
 +      while (tokens.hasMoreTokens()) {
 +        String itstring = tokens.nextToken();
 +        ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
 +        list.add(new IteratorSetting(new DataInputStream(bais)));
 +        bais.close();
 +      }
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("couldn't decode iterator settings");
 +    }
 +    return list;
 +  }
 +
 +  /**
 +   * Restricts the columns that will be mapped over for the single input table on this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param columnFamilyColumnQualifierPairs
 +   *          a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
 +   *          selected. An empty set is the default and is equivalent to scanning the all columns.
 +   * @throws IllegalArgumentException
 +   *           if the column family is null
 +   * @since 1.6.0
 +   */
 +  public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
 +    notNull(columnFamilyColumnQualifierPairs);
 +    String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
 +    conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
 +  }
 +
 +  public static String[] serializeColumns(Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
 +    notNull(columnFamilyColumnQualifierPairs);
 +    ArrayList<String> columnStrings = new ArrayList<String>(columnFamilyColumnQualifierPairs.size());
 +    for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
 +
 +      if (column.getFirst() == null)
 +        throw new IllegalArgumentException("Column family can not be null");
 +
 +      String col = Base64.encodeBase64String(TextUtil.getBytes(column.getFirst()));
 +      if (column.getSecond() != null)
 +        col += ":" + Base64.encodeBase64String(TextUtil.getBytes(column.getSecond()));
 +      columnStrings.add(col);
 +    }
 +
 +    return columnStrings.toArray(new String[0]);
 +  }
 +
 +  /**
 +   * Gets the columns to be mapped over from this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return a set of columns
 +   * @since 1.6.0
 +   * @see #fetchColumns(Class, Configuration, Collection)
 +   */
 +  public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass, Configuration conf) {
 +    notNull(conf);
 +    String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
 +    List<String> serialized = new ArrayList<String>();
 +    if (confValue != null) {
 +      // Split and include any trailing empty strings to allow empty column families
 +      for (String val : confValue.split(",", -1)) {
 +        serialized.add(val);
 +      }
 +    }
 +    return deserializeFetchedColumns(serialized);
 +  }
 +
 +  public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
 +    Set<Pair<Text,Text>> columns = new HashSet<Pair<Text,Text>>();
 +
 +    if (null == serialized) {
 +      return columns;
 +    }
 +
 +    for (String col : serialized) {
 +      int idx = col.indexOf(":");
-       Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8)));
-       Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8)));
++      Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(UTF_8)));
++      Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(UTF_8)));
 +      columns.add(new Pair<Text,Text>(cf, cq));
 +    }
 +    return columns;
 +  }
 +
 +  /**
 +   * Encode an iterator on the input for the single input table associated with this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param cfg
 +   *          the configuration of the iterator
 +   * @throws IllegalArgumentException
 +   *           if the iterator can't be serialized into the configuration
 +   * @since 1.6.0
 +   */
 +  public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    String newIter;
 +    try {
 +      cfg.write(new DataOutputStream(baos));
 +      newIter = Base64.encodeBase64String(baos.toByteArray());
 +      baos.close();
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("unable to serialize IteratorSetting");
 +    }
 +
 +    String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
 +    String iterators = conf.get(confKey);
 +    // No iterators specified yet, create a new string
 +    if (iterators == null || iterators.isEmpty()) {
 +      iterators = newIter;
 +    } else {
 +      // append the next iterator & reset
 +      iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
 +    }
 +    // Store the iterators w/ the job
 +    conf.set(confKey, iterators);
 +  }
 +
 +  /**
 +   * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
 +   * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
 +   * 
 +   * <p>
 +   * By default, this feature is <b>enabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @see #setRanges(Class, Configuration, Collection)
 +   * @since 1.6.0
 +   */
 +  public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has auto-adjust ranges enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return false if the feature is disabled, true otherwise
 +   * @since 1.6.0
 +   * @see #setAutoAdjustRanges(Class, Configuration, boolean)
 +   */
 +  public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
 +  }
 +
 +  /**
 +   * Controls the use of the {@link IsolatedScanner} in this job.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has isolation enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setScanIsolation(Class, Configuration, boolean)
 +   */
 +  public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
 +  }
 +
 +  /**
 +   * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
 +   * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration uses local iterators.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setLocalIterators(Class, Configuration, boolean)
 +   */
 +  public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
 +  }
 +
 +  /**
 +   * <p>
 +   * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
 +   * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
 +   * fail.
 +   * 
 +   * <p>
 +   * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
 +   * 
 +   * <p>
 +   * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
 +   * on the mapper's classpath.
 +   * 
 +   * <p>
 +   * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
 +   * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
 +   * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
 +   * 
 +   * <p>
 +   * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
 +   * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has the offline table scan feature enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setOfflineTableScan(Class, Configuration, boolean)
 +   */
 +  public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
 +  }
 +
 +  /**
 +   * Sets configurations for multiple tables at a time.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param configs
 +   *          an array of {@link InputTableConfig} objects to associate with the job
 +   * @since 1.6.0
 +   */
 +  public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String,InputTableConfig> configs) {
 +    MapWritable mapWritable = new MapWritable();
 +    for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
 +      mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
 +
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    try {
 +      mapWritable.write(new DataOutputStream(baos));
 +    } catch (IOException e) {
 +      throw new IllegalStateException("Table configuration could not be serialized.");
 +    }
 +
 +    String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
 +    conf.set(confKey, Base64.encodeBase64String(baos.toByteArray()));
 +  }
 +
 +  /**
 +   * Returns all {@link InputTableConfig} objects associated with this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return all of the table query configs for the job
 +   * @since 1.6.0
 +   */
 +  public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
 +    Map<String,InputTableConfig> configs = new HashMap<String,InputTableConfig>();
 +    Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
 +    if (defaultConfig != null)
 +      configs.put(defaultConfig.getKey(), defaultConfig.getValue());
 +    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
 +    MapWritable mapWritable = new MapWritable();
 +    if (configString != null) {
 +      try {
 +        byte[] bytes = Base64.decodeBase64(configString.getBytes());
 +        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
 +        mapWritable.readFields(new DataInputStream(bais));
 +        bais.close();
 +      } catch (IOException e) {
 +        throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
 +      }
 +    }
 +    for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
 +      configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());
 +
 +    return configs;
 +  }
 +
 +  /**
 +   * Returns the {@link InputTableConfig} for the given table
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table name for which to fetch the table query config
 +   * @return the table query config for the given table name (if it exists) and null if it does not
 +   * @since 1.6.0
 +   */
 +  public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf, String tableName) {
 +    Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
 +    return queryConfigs.get(tableName);
 +  }
 +
 +  /**
 +   * Initializes an Accumulo {@link TabletLocator} based on the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableId
 +   *          The table id for which to initialize the {@link TabletLocator}
 +   * @return an Accumulo tablet locator
 +   * @throws TableNotFoundException
 +   *           if the table name set on the configuration doesn't exist
 +   * @since 1.6.0
 +   */
 +  public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf, String tableId) throws TableNotFoundException {
 +    String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
 +    if ("MockInstance".equals(instanceType))
 +      return new MockTabletLocator();
 +    Instance instance = getInstance(implementingClass, conf);
 +    return TabletLocator.getLocator(instance, new Text(tableId));
 +  }
 +
 +  // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
 +  /**
 +   * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @throws IOException
 +   *           if the context is improperly configured
 +   * @since 1.6.0
 +   */
 +  public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
 +
 +    Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
 +    if (!isConnectorInfoSet(implementingClass, conf))
 +      throw new IOException("Input info has not been set.");
 +    String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
 +    if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
 +      throw new IOException("Instance info has not been set.");
 +    // validate that we can connect as configured
 +    try {
 +      String principal = getPrincipal(implementingClass, conf);
 +      AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
 +      Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
 +      if (!c.securityOperations().authenticateUser(principal, token))
 +        throw new IOException("Unable to authenticate user");
 +
 +      if (getInputTableConfigs(implementingClass, conf).size() == 0)
 +        throw new IOException("No table set.");
 +
 +      for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
 +        if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
 +          throw new IOException("Unable to access table");
 +      }
 +      for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
 +        InputTableConfig tableConfig = tableConfigEntry.getValue();
 +        if (!tableConfig.shouldUseLocalIterators()) {
 +          if (tableConfig.getIterators() != null) {
 +            for (IteratorSetting iter : tableConfig.getIterators()) {
 +              if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
 +                throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
 +            }
 +          }
 +        }
 +      }
 +    } catch (AccumuloException e) {
 +      throw new IOException(e);
 +    } catch (AccumuloSecurityException e) {
 +      throw new IOException(e);
 +    } catch (TableNotFoundException e) {
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  /**
 +   * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the configuration based on the properties set using the single-table
 +   * input methods.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop instance for which to retrieve the configuration
 +   * @return the config object built from the single input table properties set on the job
 +   * @since 1.6.0
 +   */
 +  protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(Class<?> implementingClass, Configuration conf) {
 +    String tableName = getInputTableName(implementingClass, conf);
 +    if (tableName != null) {
 +      InputTableConfig queryConfig = new InputTableConfig();
 +      List<IteratorSetting> itrs = getIterators(implementingClass, conf);
 +      if (itrs != null)
 +        queryConfig.setIterators(itrs);
 +      Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
 +      if (columns != null)
 +        queryConfig.fetchColumns(columns);
 +      List<Range> ranges = null;
 +      try {
 +        ranges = getRanges(implementingClass, conf);
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      if (ranges != null)
 +        queryConfig.setRanges(ranges);
 +
 +      queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)).setUseIsolatedScanners(isIsolated(implementingClass, conf))
 +          .setUseLocalIterators(usesLocalIterators(implementingClass, conf)).setOfflineScan(isOfflineScan(implementingClass, conf));
 +      return Maps.immutableEntry(tableName, queryConfig);
 +    }
 +    return null;
 +  }
 +
 +  public static Map<String,Map<KeyExtent,List<Range>>> binOffline(String tableId, List<Range> ranges, Instance instance, Connector conn)
 +      throws AccumuloException, TableNotFoundException {
 +    Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
 +
 +    if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
 +      Tables.clearCache(instance);
 +      if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
 +        throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
 +      }
 +    }
 +
 +    for (Range range : ranges) {
 +      Text startRow;
 +
 +      if (range.getStartKey() != null)
 +        startRow = range.getStartKey().getRow();
 +      else
 +        startRow = new Text();
 +
 +      Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
 +      Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +      MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
 +      scanner.setRange(metadataRange);
 +
 +      RowIterator rowIter = new RowIterator(scanner);
 +      KeyExtent lastExtent = null;
 +      while (rowIter.hasNext()) {
 +        Iterator<Map.Entry<Key,Value>> row = rowIter.next();
 +        String last = "";
 +        KeyExtent extent = null;
 +        String location = null;
 +
 +        while (row.hasNext()) {
 +          Map.Entry<Key,Value> entry = row.next();
 +          Key key = entry.getKey();
 +
 +          if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
 +            last = entry.getValue().toString();
 +          }
 +
 +          if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
 +              || key.getColumnFamily().equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
 +            location = entry.getValue().toString();
 +          }
 +
 +          if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
 +            extent = new KeyExtent(key.getRow(), entry.getValue());
 +          }
 +
 +        }
 +
 +        if (location != null)
 +          return null;
 +
 +        if (!extent.getTableId().toString().equals(tableId)) {
 +          throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
 +        }
 +
 +        if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
 +          throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
 +        }
 +
 +        Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
 +        if (tabletRanges == null) {
 +          tabletRanges = new HashMap<KeyExtent,List<Range>>();
 +          binnedRanges.put(last, tabletRanges);
 +        }
 +
 +        List<Range> rangeList = tabletRanges.get(extent);
 +        if (rangeList == null) {
 +          rangeList = new ArrayList<Range>();
 +          tabletRanges.put(extent, rangeList);
 +        }
 +
 +        rangeList.add(range);
 +
 +        if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
 +          break;
 +        }
 +
 +        lastExtent = extent;
 +      }
 +
 +    }
 +    return binnedRanges;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
index 0ba42cd,0000000..26680dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
@@@ -1,204 -1,0 +1,205 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.hadoop.conf.Configuration;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class OutputConfigurator extends ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link BatchWriter}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum WriteOpts {
 +    DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
 +  }
 +
 +  /**
 +   * Configuration keys for various features.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum Features {
 +    CAN_CREATE_TABLES, SIMULATION_MODE
 +  }
 +
 +  /**
 +   * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
 +   * underscores.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table to use when the tablename is null in the write call
 +   * @since 1.6.0
 +   */
 +  public static void setDefaultTableName(Class<?> implementingClass, Configuration conf, String tableName) {
 +    if (tableName != null)
 +      conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
 +  }
 +
 +  /**
 +   * Gets the default table name from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the default table name
 +   * @since 1.6.0
 +   * @see #setDefaultTableName(Class, Configuration, String)
 +   */
 +  public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
 +  }
 +
 +  /**
 +   * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
 +   * used. Setting the configuration multiple times overwrites any previous configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param bwConfig
 +   *          the configuration for the {@link BatchWriter}
 +   * @since 1.6.0
 +   */
 +  public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf, BatchWriterConfig bwConfig) {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    String serialized;
 +    try {
 +      bwConfig.write(new DataOutputStream(baos));
-       serialized = new String(baos.toByteArray(), Constants.UTF8);
++      serialized = new String(baos.toByteArray(), UTF_8);
 +      baos.close();
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
 +    }
 +    conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
 +  }
 +
 +  /**
 +   * Gets the {@link BatchWriterConfig} settings.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the configuration object
 +   * @since 1.6.0
 +   * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
 +   */
 +  public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
 +    String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
 +    BatchWriterConfig bwConfig = new BatchWriterConfig();
 +    if (serialized == null || serialized.isEmpty()) {
 +      return bwConfig;
 +    } else {
 +      try {
-         ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(Constants.UTF8));
++        ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
 +        bwConfig.readFields(new DataInputStream(bais));
 +        bais.close();
 +        return bwConfig;
 +      } catch (IOException e) {
 +        throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setCreateTables(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether tables are permitted to be created as needed.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is disabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setCreateTables(Class, Configuration, boolean)
 +   */
 +  public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
 +  }
 +
 +  /**
 +   * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setSimulationMode(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether this feature is enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setSimulationMode(Class, Configuration, boolean)
 +   */
 +  public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 1541fae,0753282..9e26672
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@@ -26,11 -28,10 +28,10 @@@ import java.util.Arrays
  import java.util.Scanner;
  import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
 -import org.apache.commons.codec.binary.Base64;
 +import org.apache.accumulo.core.client.mapreduce.lib.impl.DistributedCacheHelper;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.hadoop.conf.Configurable;
  import org.apache.hadoop.conf.Configuration;
 -import org.apache.hadoop.filecache.DistributedCache;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.Text;
  import org.apache.hadoop.io.Writable;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
index 0d3aca2,d95a647..95fcf53
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
@@@ -16,20 -16,18 +16,21 @@@
   */
  package org.apache.accumulo.core.client.mock;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.io.ByteArrayInputStream;
 -import java.io.File;
  import java.io.IOException;
  import java.io.InputStream;
 +import java.io.OutputStream;
  import java.io.Writer;
  
 -import jline.ConsoleReader;
 +import org.apache.commons.io.output.WriterOutputStream;
  
 -import org.apache.accumulo.core.util.shell.Shell;
  import org.apache.commons.cli.CommandLine;
 +import jline.console.ConsoleReader;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.shell.Shell;
 +import org.apache.accumulo.core.util.shell.ShellOptionsJC;
  
  /**
   * An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes.
@@@ -38,29 -36,12 +39,29 @@@ public class MockShell extends Shell 
    private static final String NEWLINE = "\n";
    
    protected InputStream in;
 -  protected Writer writer;
 -  
 -  public MockShell(InputStream in, Writer writer) throws IOException {
 +  protected OutputStream out;
 +
 +  /**
 +   * Will only be set if you use either the Writer constructor or the setWriter(Writer) method
 +   * @deprecated since 1.6.0; use out
 +   */
 +  @Deprecated
 +  protected Writer writer = null;
 +
 +  public MockShell(InputStream in, OutputStream out) throws IOException {
      super();
      this.in = in;
 -    this.writer = writer;
 +    this.out = out;
 +    // we presume they don't use the writer field unless they use the other constructor.
 +  }
 +
 +  /**
 +   * @deprecated since 1.6.0; use OutputStream version
 +   */
 +  @Deprecated
 +  public MockShell(InputStream in, Writer out) throws IOException {
-     this(in, new WriterOutputStream(out, Constants.UTF8.name()));
++    this(in, new WriterOutputStream(out, UTF_8.name()));
 +    this.writer = out;
    }
    
    public boolean config(String... args) {
@@@ -107,7 -79,7 +108,7 @@@
        printInfo();
      
      if (execFile != null) {
-       java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
 -      java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++      java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
        try {
          while (scanner.hasNextLine() && !hasExited()) {
            execCommand(scanner.nextLine(), true, isVerbose());
@@@ -146,20 -118,11 +147,20 @@@
    }
    
    /**
 -   * @param writer
 -   *          the writer to set
 +   * @param out
 +   *          the output stream to set
 +   */
 +  public void setConsoleWriter(OutputStream out) {
 +    this.out = out;
 +  }
 +
 +  /**
 +   * @deprecated since 1.6.0; use the OutputStream version
     */
 -  public void setConsoleWriter(Writer writer) {
 -    this.writer = writer;
 +  @Deprecated
 +  public void setConsoleWriter(Writer out) {
-     setConsoleWriter(new WriterOutputStream(out, Constants.UTF8.name()));
++    setConsoleWriter(new WriterOutputStream(out, UTF_8.name()));
 +    this.writer = out;
    }
    
    /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index bd9feba,05aa0c3..4d73979
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@@ -124,14 -123,10 +125,14 @@@ public class PasswordToken implements A
        throw new RuntimeException(e);
      }
    }
 -  
 -  private void setPassword(CharBuffer charBuffer) {
 +
 +  protected void setPassword(byte[] password) {
 +    this.password = Arrays.copyOf(password, password.length);
 +  }
 +
 +  protected void setPassword(CharBuffer charBuffer) {
      // encode() kicks back a C-string, which is not compatible with the old passwording system
-     ByteBuffer bb = Constants.UTF8.encode(charBuffer);
+     ByteBuffer bb = UTF_8.encode(charBuffer);
      // create array using byter buffer length
      this.password = new byte[bb.remaining()];
      bb.get(this.password);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 41a1f3b,0000000..618ff7e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -1,356 -1,0 +1,357 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.PrintStream;
 +import java.io.UnsupportedEncodingException;
 +import java.util.ArrayList;
 +import java.util.TreeMap;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * This class generates documentation to inform users of the available configuration properties in a presentable form.
 + */
 +class ConfigurationDocGen {
 +  private abstract class Format {
 +
 +    abstract void beginSection(String section);
 +
 +    void endSection() {}
 +
 +    void generate() {
 +      pageHeader();
 +
 +      beginSection("Available Properties");
 +      for (Property prefix : prefixes) {
 +        if (!prefix.isExperimental()) {
 +          prefixSection(prefix);
 +          for (Property prop : sortedProps.values()) {
 +            if (!prop.isExperimental()) {
 +              property(prefix, prop);
 +            }
 +          }
 +        }
 +      }
 +      endSection();
 +
 +      beginSection("Property Types");
 +      propertyTypeDescriptions();
 +      endSection();
 +
 +      pageFooter();
 +      doc.close();
 +    }
 +
 +    abstract String getExt();
 +
 +    void pageFooter() {}
 +
 +    // read static header content from resources and output
 +    void pageHeader() {
 +      appendResource("config-header." + getExt());
 +      doc.println();
 +    }
 +
 +    abstract void prefixSection(Property prefix);
 +
 +    abstract void property(Property prefix, Property prop);
 +
 +    abstract void propertyTypeDescriptions();
 +
 +    abstract String sanitize(String str);
 +
 +  }
 +
 +  private class HTML extends Format {
 +    private boolean highlight;
 +
 +    private void beginRow() {
 +      doc.println(startTag("tr", highlight ? "class='highlight'" : null));
 +      highlight = !highlight;
 +    }
 +
 +    @Override
 +    void beginSection(String section) {
 +      doc.println(t("h1", section, null));
 +      highlight = true;
 +      doc.println("<table>");
 +    }
 +
 +    private String t(String tag, String cell, String options) {
 +      return startTag(tag, options) + cell + "</" + tag + ">";
 +    }
 +
 +    private void cellData(String cell, String options) {
 +      doc.println(t("td", cell, options));
 +    }
 +
 +    private void columnNames(String... names) {
 +      beginRow();
 +      for (String s : names)
 +        doc.println(t("th", s, null));
 +      endRow();
 +    }
 +
 +    private void endRow() {
 +      doc.println("</tr>");
 +    }
 +
 +    @Override
 +    void endSection() {
 +      doc.println("</table>");
 +    }
 +
 +    @Override
 +    String getExt() {
 +      return "html";
 +    }
 +
 +    @Override
 +    void pageFooter() {
 +      doc.println("</body>");
 +      doc.println("</html>");
 +    }
 +
 +    @Override
 +    void pageHeader() {
 +      super.pageHeader();
 +      doc.println("<p>Jump to: ");
 +      String delimiter = "";
 +      for (Property prefix : prefixes) {
 +        if (!prefix.isExperimental()) {
 +          doc.print(delimiter + "<a href='#" + prefix.name() + "'>" + prefix.getKey() + "*</a>");
 +          delimiter = "&nbsp;|&nbsp;";
 +        }
 +      }
 +      doc.println("</p>");
 +    }
 +
 +    @Override
 +    void prefixSection(Property prefix) {
 +      beginRow();
 +      doc.println(t("td", t("span", prefix.getKey() + '*', "id='" + prefix.name() + "' class='large'"), "colspan='5'"
 +          + (prefix.isDeprecated() ? " class='deprecated'" : "")));
 +      endRow();
 +      beginRow();
 +      doc.println(t("td", (prefix.isDeprecated() ? t("b", t("i", "Deprecated. ", null), null) : "") + sanitize(prefix.getDescription()), "colspan='5'"
 +          + (prefix.isDeprecated() ? " class='deprecated'" : "")));
 +      endRow();
 +
 +      switch (prefix) {
 +        case TABLE_CONSTRAINT_PREFIX:
 +          break;
 +        case TABLE_ITERATOR_PREFIX:
 +          break;
 +        case TABLE_LOCALITY_GROUP_PREFIX:
 +          break;
 +        case TABLE_COMPACTION_STRATEGY_PREFIX:
 +          break;
 +        default:
 +          columnNames("Property", "Type", "ZooKeeper Mutable", "Default Value", "Description");
 +          break;
 +      }
 +    }
 +
 +    @Override
 +    void property(Property prefix, Property prop) {
 +      boolean isDeprecated = prefix.isDeprecated() || prop.isDeprecated();
 +      if (prop.getKey().startsWith(prefix.getKey())) {
 +        beginRow();
 +        cellData(prop.getKey(), isDeprecated ? "class='deprecated'" : null);
 +        cellData("<b><a href='#" + prop.getType().name() + "'>" + prop.getType().toString().replaceAll(" ", "&nbsp;") + "</a></b>",
 +            isDeprecated ? "class='deprecated'" : null);
 +        cellData(isZooKeeperMutable(prop), isDeprecated ? "class='deprecated'" : null);
 +        cellData("<pre>" + (prop.getRawDefaultValue().isEmpty() ? "&nbsp;" : sanitize(prop.getRawDefaultValue().replaceAll(" ", "&nbsp;"))) + "</pre>",
 +            isDeprecated ? "class='deprecated'" : null);
 +        cellData((isDeprecated ? "<b><i>Deprecated.</i></b> " : "") + sanitize(prop.getDescription()), isDeprecated ? "class='deprecated'" : null);
 +        endRow();
 +      }
 +
 +    }
 +
 +    @Override
 +    void propertyTypeDescriptions() {
 +      columnNames("Property Type", "Description");
 +      for (PropertyType type : PropertyType.values()) {
 +        if (type == PropertyType.PREFIX)
 +          continue;
 +        beginRow();
 +        cellData("<h3 id='" + type.name() + "'>" + type + "</h3>", null);
 +        cellData(type.getFormatDescription(), null);
 +        endRow();
 +      }
 +    }
 +
 +    @Override
 +    String sanitize(String str) {
 +      return str.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replaceAll("(?:\r\n|\r|\n)", "<br />");
 +    }
 +
 +    private String startTag(String tag, String options) {
 +      return "<" + tag + (options != null ? " " + options : "") + ">";
 +    }
 +
 +  }
 +
 +  private class LaTeX extends Format {
 +    @Override
 +    void beginSection(String section) {
 +      doc.println("\\section{" + section + "}");
 +    }
 +
 +    @Override
 +    String getExt() {
 +      return "tex";
 +    }
 +
 +    @Override
 +    void prefixSection(Property prefix) {
 +      boolean depr = prefix.isDeprecated();
 +      doc.println("\\subsection{" + prefix.getKey() + "*}" + (depr ? " (Deprecated)" : ""));
 +      doc.println((depr ? "\\sout{\\textit{Deprecated.} " : "") + sanitize(prefix.getDescription()) + (depr ? "}" : ""));
 +      doc.println();
 +    }
 +
 +    @Override
 +    void property(Property prefix, Property prop) {
 +      boolean depr = prefix.isDeprecated() || prop.isDeprecated();
 +      if (prop.getKey().startsWith(prefix.getKey())) {
 +        doc.println("\\subsubsection{" + prop.getKey() + "}");
 +        doc.println(strike((depr ? "\\textit{Deprecated.} " : "") + sanitize(prop.getDescription()), depr));
 +        doc.println();
 +        doc.println(strike("\\textit{Type:} " + prop.getType().name() + "\\\\", depr));
 +        doc.println(strike("\\textit{ZooKeeper Mutable:} " + isZooKeeperMutable(prop) + "\\\\", depr));
 +        doc.println(strike("\\textit{Default Value:} " + sanitize(prop.getRawDefaultValue()), depr));
 +        doc.println();
 +      }
 +    }
 +
 +    private String strike(String s, boolean isDeprecated) {
 +      return (isDeprecated ? "\\sout{" : "") + s + (isDeprecated ? "}" : "");
 +    }
 +
 +    @Override
 +    void propertyTypeDescriptions() {
 +      for (PropertyType type : PropertyType.values()) {
 +        if (type == PropertyType.PREFIX)
 +          continue;
 +        doc.println("\\subsection{" + sanitize(type.toString()) + "}");
 +        doc.println(sanitize(type.getFormatDescription()));
 +        doc.println();
 +      }
 +    }
 +
 +    @Override
 +    String sanitize(String str) {
 +      str = str.replace("\\", "\\textbackslash{}");
 +      str = str.replace("~", "\\textasciitilde{}");
 +      str = str.replace("^", "\\textasciicircum");
 +
 +      str = str.replace("&", "\\&");
 +      str = str.replace("%", "\\%");
 +      str = str.replace("$", "\\$");
 +      str = str.replace("#", "\\#");
 +      str = str.replace("_", "\\_");
 +      str = str.replace("{", "\\{");
 +      str = str.replace("}", "\\}");
 +
 +      str = str.replaceAll("(?:\r\n|\r|\n)", "\\\\\\\\\n");
 +      return str;
 +    }
 +  }
 +
 +  private static Logger log = Logger.getLogger(ConfigurationDocGen.class);
 +
 +  private PrintStream doc;
 +
 +  private final ArrayList<Property> prefixes;
 +
 +  private final TreeMap<String,Property> sortedProps;
 +
 +  ConfigurationDocGen(PrintStream doc) {
 +    this.doc = doc;
 +    this.prefixes = new ArrayList<Property>();
 +    this.sortedProps = new TreeMap<String,Property>();
 +
 +    for (Property prop : Property.values()) {
 +      if (prop.isExperimental())
 +        continue;
 +
 +      if (prop.getType() == PropertyType.PREFIX)
 +        this.prefixes.add(prop);
 +      else
 +        this.sortedProps.put(prop.getKey(), prop);
 +    }
 +  }
 +
 +  private void appendResource(String resourceName) {
 +    InputStream data = ConfigurationDocGen.class.getResourceAsStream(resourceName);
 +    if (data != null) {
 +      byte[] buffer = new byte[1024];
 +      int n;
 +      try {
 +        while ((n = data.read(buffer)) > 0)
-           doc.print(new String(buffer, 0, n, Constants.UTF8));
++          doc.print(new String(buffer, 0, n, UTF_8));
 +      } catch (IOException e) {
 +        e.printStackTrace();
 +        return;
 +      } finally {
 +        try {
 +          data.close();
 +        } catch (IOException ex) {
 +          log.error(ex, ex);
 +        }
 +      }
 +    }
 +  }
 +
 +  private String isZooKeeperMutable(Property prop) {
 +    if (!Property.isValidZooPropertyKey(prop.getKey()))
 +      return "no";
 +    if (Property.isFixedZooPropertyKey(prop))
 +      return "yes but requires restart of the " + prop.getKey().split("[.]")[0];
 +    return "yes";
 +  }
 +
 +  void generateHtml() {
 +    new HTML().generate();
 +  }
 +
 +  void generateLaTeX() {
 +    new LaTeX().generate();
 +  }
 +
 +  /*
 +   * Generate documentation for conf/accumulo-site.xml file usage
 +   */
 +  public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
 +    if (args.length == 2 && args[0].equals("--generate-html")) {
-       new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateHtml();
++      new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateHtml();
 +    } else if (args.length == 2 && args[0].equals("--generate-latex")) {
-       new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateLaTeX();
++      new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateLaTeX();
 +    } else {
 +      throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index ca769cb,7c4f66e..437590f
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@@ -16,11 -16,10 +16,11 @@@
   */
  package org.apache.accumulo.core.data;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.io.Serializable;
 +import java.nio.ByteBuffer;
  
- import org.apache.accumulo.core.Constants;
- 
  public class ArrayByteSequence extends ByteSequence implements Serializable {
    
    private static final long serialVersionUID = 1L;
@@@ -48,22 -47,9 +48,22 @@@
    }
    
    public ArrayByteSequence(String s) {
-     this(s.getBytes(Constants.UTF8));
+     this(s.getBytes(UTF_8));
    }
    
 +  public ArrayByteSequence(ByteBuffer buffer) {
 +    this.length = buffer.remaining();
 +
 +    if (buffer.hasArray()) {
 +      this.data = buffer.array();
 +      this.offset = buffer.position();
 +    } else {
 +      this.data = new byte[length];
 +      this.offset = 0;
 +      buffer.get(data);
 +    }
 +  }
 +
    @Override
    public byte byteAt(int i) {
      


[18/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index bfd4818,0000000..e6e31eb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -1,238 -1,0 +1,239 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Arrays;
 +import java.util.HashSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.hadoop.io.Text;
 +
 +/**
 + * Conditions that must be met on a particular column in a row.
 + * 
 + * @since 1.6.0
 + */
 +public class Condition {
 +  
 +  private ByteSequence cf;
 +  private ByteSequence cq;
 +  private ByteSequence cv;
 +  private ByteSequence val;
 +  private Long ts;
 +  private IteratorSetting iterators[] = new IteratorSetting[0];
 +  private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
 +  
 +
 +  public Condition(CharSequence cf, CharSequence cq) {
 +    ArgumentChecker.notNull(cf, cq);
-     this.cf = new ArrayByteSequence(cf.toString().getBytes(Constants.UTF8));
-     this.cq = new ArrayByteSequence(cq.toString().getBytes(Constants.UTF8));
++    this.cf = new ArrayByteSequence(cf.toString().getBytes(UTF_8));
++    this.cq = new ArrayByteSequence(cq.toString().getBytes(UTF_8));
 +    this.cv = EMPTY;
 +  }
 +  
 +  public Condition(byte[] cf, byte[] cq) {
 +    ArgumentChecker.notNull(cf, cq);
 +    this.cf = new ArrayByteSequence(cf);
 +    this.cq = new ArrayByteSequence(cq);
 +    this.cv = EMPTY;
 +  }
 +
 +  public Condition(Text cf, Text cq) {
 +    ArgumentChecker.notNull(cf, cq);
 +    this.cf = new ArrayByteSequence(cf.getBytes(), 0, cf.getLength());
 +    this.cq = new ArrayByteSequence(cq.getBytes(), 0, cq.getLength());
 +    this.cv = EMPTY;
 +  }
 +
 +  public Condition(ByteSequence cf, ByteSequence cq) {
 +    ArgumentChecker.notNull(cf, cq);
 +    this.cf = cf;
 +    this.cq = cq;
 +    this.cv = EMPTY;
 +  }
 +
 +  public ByteSequence getFamily() {
 +    return cf;
 +  }
 +  
 +  public ByteSequence getQualifier() {
 +    return cq;
 +  }
 +
 +  /**
 +   * Sets the version for the column to check. If this is not set then the latest column will be checked, unless iterators do something different.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setTimestamp(long ts) {
 +    this.ts = ts;
 +    return this;
 +  }
 +  
 +  public Long getTimestamp() {
 +    return ts;
 +  }
 +
 +  /**
 +   * see {@link #setValue(byte[])}
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(CharSequence value) {
 +    ArgumentChecker.notNull(value);
-     this.val = new ArrayByteSequence(value.toString().getBytes(Constants.UTF8));
++    this.val = new ArrayByteSequence(value.toString().getBytes(UTF_8));
 +    return this;
 +  }
 +
 +  /**
 +   * This method sets the expected value of a column. Inorder for the condition to pass the column must exist and have this value. If a value is not set, then
 +   * the column must be absent for the condition to pass.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(byte[] value) {
 +    ArgumentChecker.notNull(value);
 +    this.val = new ArrayByteSequence(value);
 +    return this;
 +  }
 +  
 +  /**
 +   * see {@link #setValue(byte[])}
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(Text value) {
 +    ArgumentChecker.notNull(value);
 +    this.val = new ArrayByteSequence(value.getBytes(), 0, value.getLength());
 +    return this;
 +  }
 +  
 +  /**
 +   * see {@link #setValue(byte[])}
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setValue(ByteSequence value) {
 +    ArgumentChecker.notNull(value);
 +    this.val = value;
 +    return this;
 +  }
 +
 +  public ByteSequence getValue() {
 +    return val;
 +  }
 +
 +  /**
 +   * Sets the visibility for the column to check. If not set it defaults to empty visibility.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setVisibility(ColumnVisibility cv) {
 +    ArgumentChecker.notNull(cv);
 +    this.cv = new ArrayByteSequence(cv.getExpression());
 +    return this;
 +  }
 +
 +  public ByteSequence getVisibility() {
 +    return cv;
 +  }
 +
 +  /**
 +   * Set iterators to use when reading the columns value. These iterators will be applied in addition to the iterators configured for the table. Using iterators
 +   * its possible to test other conditions, besides equality and absence, like less than. On the server side the iterators will be seeked using a range that
 +   * covers only the family, qualifier, and visibility (if the timestamp is set then it will be used to narrow the range). Value equality will be tested using
 +   * the first entry returned by the iterator stack.
 +   * 
 +   * @return returns this
 +   */
 +
 +  public Condition setIterators(IteratorSetting... iterators) {
 +    ArgumentChecker.notNull(iterators);
 +    
 +    if (iterators.length > 1) {
 +      HashSet<String> names = new HashSet<String>();
 +      HashSet<Integer> prios = new HashSet<Integer>();
 +      
 +      for (IteratorSetting iteratorSetting : iterators) {
 +        if (!names.add(iteratorSetting.getName()))
 +          throw new IllegalArgumentException("iterator name used more than once " + iteratorSetting.getName());
 +        if (!prios.add(iteratorSetting.getPriority()))
 +          throw new IllegalArgumentException("iterator priority used more than once " + iteratorSetting.getPriority());
 +      }
 +    }
 +    
 +    this.iterators = iterators;
 +    return this;
 +  }
 +
 +  public IteratorSetting[] getIterators() {
 +    return iterators;
 +  }
 +
 +  @Override
 +  public boolean equals(Object o) {
 +    if (o == this) {
 +      return true;
 +    }
 +    if (o == null || !(o instanceof Condition)) {
 +      return false;
 +    }
 +    Condition c = (Condition) o;
 +    if (!(c.cf.equals(cf))) {
 +      return false;
 +    }
 +    if (!(c.cq.equals(cq))) {
 +      return false;
 +    }
 +    if (!(c.cv.equals(cv))) {
 +      return false;
 +    }
 +    if (!(c.val == null ? val == null : c.val.equals(val))) {
 +      return false;
 +    }
 +    if (!(c.ts == null ? ts == null : c.ts.equals(ts))) {
 +      return false;
 +    }
 +    if (!(Arrays.equals(c.iterators, iterators))) {
 +      return false;
 +    }
 +    return true;
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    int result = 17;
 +    result = 31 * result + cf.hashCode();
 +    result = 31 * result + cq.hashCode();
 +    result = 31 * result + cv.hashCode();
 +    result = 31 * result + (val == null ? 0 : val.hashCode());
 +    result = 31 * result + (ts == null ? 0 : ts.hashCode());
 +    result = 31 * result + Arrays.hashCode(iterators);
 +    return result;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Value.java
index 11e60e1,bc1f71d..d256107
--- a/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@@ -25,9 -26,6 +26,8 @@@ import java.io.IOException
  import java.nio.ByteBuffer;
  import java.util.List;
  
 +import com.google.common.base.Preconditions;
 +
- import org.apache.accumulo.core.Constants;
  import org.apache.hadoop.io.BytesWritable;
  import org.apache.hadoop.io.WritableComparable;
  import org.apache.hadoop.io.WritableComparator;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
index 8006114,0b220ee..7073d11
--- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
@@@ -24,10 -26,7 +26,9 @@@ import java.util.Collections
  import java.util.HashSet;
  import java.util.Iterator;
  import java.util.List;
 +import java.util.Set;
 +import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.data.ArrayByteSequence;
  import org.apache.accumulo.core.data.ByteSequence;
  import org.apache.accumulo.core.util.ArgumentChecker;
@@@ -195,13 -166,12 +196,13 @@@ public class Authorizations implements 
    }
  
    /**
 -   * Retrieve a serialized form of the underlying set of authorizations.
 +   * Returns a serialized form of these authorizations.
     * 
 -   * @see #Authorizations(byte[])
 +   * @return serialized form of these authorizations, as a string encoded in UTF-8
 +   * @see #serialize()
     */
    public byte[] getAuthorizationsArray() {
-     return serialize().getBytes(Constants.UTF8);
+     return serialize().getBytes(UTF_8);
    }
  
    /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index e76b0ef,66b68ab..e8f4e97
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -243,9 -223,9 +244,9 @@@ public class ColumnVisibility 
    /*
     * Walks an expression's AST and appends a string representation to a supplied StringBuilder. This method adds parens where necessary.
     */
 -  private static void stringify(Node root, byte[] expression, StringBuilder out) {
 +  public static void stringify(Node root, byte[] expression, StringBuilder out) {
      if (root.type == NodeType.TERM) {
-       out.append(new String(expression, root.start, root.end - root.start, Constants.UTF8));
+       out.append(new String(expression, root.start, root.end - root.start, UTF_8));
      } else {
        String sep = "";
        for (Node c : root.children) {
@@@ -314,12 -293,12 +315,12 @@@
        while (index < expression.length) {
          switch (expression[index++]) {
            case '&': {
 -            expr = processTerm(termStart, index - 1, expr, expression);
 +            expr = processTerm(subtermStart, index - 1, expr, expression);
              if (result != null) {
                if (!result.type.equals(NodeType.AND))
-                 throw new BadArgumentException("cannot mix & and |", new String(expression, Constants.UTF8), index - 1);
+                 throw new BadArgumentException("cannot mix & and |", new String(expression, UTF_8), index - 1);
              } else {
 -              result = new Node(NodeType.AND);
 +              result = new Node(NodeType.AND, wholeTermStart);
              }
              result.add(expr);
              expr = null;
@@@ -328,12 -307,12 +329,12 @@@
              break;
            }
            case '|': {
 -            expr = processTerm(termStart, index - 1, expr, expression);
 +            expr = processTerm(subtermStart, index - 1, expr, expression);
              if (result != null) {
                if (!result.type.equals(NodeType.OR))
-                 throw new BadArgumentException("cannot mix | and &", new String(expression, Constants.UTF8), index - 1);
+                 throw new BadArgumentException("cannot mix | and &", new String(expression, UTF_8), index - 1);
              } else {
 -              result = new Node(NodeType.OR);
 +              result = new Node(NodeType.OR, wholeTermStart);
              }
              result.add(expr);
              expr = null;
@@@ -343,18 -322,18 +344,18 @@@
            }
            case '(': {
              parens++;
 -            if (termStart != index - 1 || expr != null)
 +            if (subtermStart != index - 1 || expr != null)
-               throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+               throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
              expr = parse_(expression);
 -            termStart = index;
 -            termComplete = false;
 +            subtermStart = index;
 +            subtermComplete = false;
              break;
            }
            case ')': {
              parens--;
 -            Node child = processTerm(termStart, index - 1, expr, expression);
 +            Node child = processTerm(subtermStart, index - 1, expr, expression);
              if (child == null && result == null)
-               throw new BadArgumentException("empty expression not allowed", new String(expression, Constants.UTF8), index);
+               throw new BadArgumentException("empty expression not allowed", new String(expression, UTF_8), index);
              if (result == null)
                return child;
              if (result.type == child.type)
@@@ -366,8 -345,8 +367,8 @@@
              return result;
            }
            case '"': {
 -            if (termStart != index - 1)
 +            if (subtermStart != index - 1)
-               throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+               throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
              
              while (index < expression.length && expression[index] != '"') {
                if (expression[index] == '\\') {
@@@ -379,20 -358,20 +380,20 @@@
              }
              
              if (index == expression.length)
-               throw new BadArgumentException("unclosed quote", new String(expression, Constants.UTF8), subtermStart);
 -              throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), termStart);
 -            
 -            if (termStart + 1 == index)
 -              throw new BadArgumentException("empty term", new String(expression, UTF_8), termStart);
++              throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), subtermStart);
              
 +            if (subtermStart + 1 == index)
-               throw new BadArgumentException("empty term", new String(expression, Constants.UTF8), subtermStart);
++              throw new BadArgumentException("empty term", new String(expression, UTF_8), subtermStart);
 + 
              index++;
              
 -            termComplete = true;
 +            subtermComplete = true;
              
              break;
            }
            default: {
 -            if (termComplete)
 +            if (subtermComplete)
-               throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+               throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
              
              byte c = expression[index - 1];
              if (!Authorizations.isValidAuthChar(c))
@@@ -514,12 -482,10 +515,12 @@@
     *   .
     * ColumnVisibility cv = new ColumnVisibility(quote(&quot;A#C&quot;) + &quot;&amp;&quot; + quote(&quot;FOO&quot;));
     * </pre>
 -   * 
 +   *
 +   * @param term term to quote
 +   * @return quoted term (unquoted if unnecessary)
     */
    public static String quote(String term) {
-     return new String(quote(term.getBytes(Constants.UTF8)), Constants.UTF8);
+     return new String(quote(term.getBytes(UTF_8)), UTF_8);
    }
    
    /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Credentials.java
index 582b4e0,31fe18d..0ecc31d
--- a/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
@@@ -16,17 -16,10 +16,18 @@@
   */
  package org.apache.accumulo.core.security;
  
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.nio.ByteBuffer;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
  import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
  import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.util.Base64;
  
  /**
   * A wrapper for internal use. This class carries the instance, principal, and authentication token for use in the public API, in a non-serialized form. This is
@@@ -53,102 -36,9 +54,102 @@@ public class Credentials 
      this.principal = principal;
      this.token = token;
    }
 -  
 -  public TCredentials toThrift() {
 -    return CredentialHelper.createSquelchError(principal, token, instance.getInstanceID());
 +
 +  /**
 +   * Gets the principal.
 +   * 
 +   * @return unique identifier for the entity (e.g. a user or service) authorized for these credentials
 +   */
 +  public String getPrincipal() {
 +    return principal;
 +  }
 +
 +  /**
 +   * Gets the authentication token.
 +   * 
 +   * @return authentication token used to prove that the principal for these credentials has been properly verified
 +   */
 +  public AuthenticationToken getToken() {
 +    return token;
 +  }
 +
 +  /**
 +   * Converts the current object to the relevant thrift type. The object returned from this contains a non-destroyable version of the
 +   * {@link AuthenticationToken}, so this should be used just before placing on the wire, and references to it should be tightly controlled.
 +   * 
 +   * @param instance
 +   *          client instance
 +   * @return Thrift credentials
 +   * @throws RuntimeException
 +   *           if the authentication token has been destroyed (expired)
 +   */
 +  public TCredentials toThrift(Instance instance) {
 +    TCredentials tCreds = new TCredentials(getPrincipal(), getToken().getClass().getName(),
 +        ByteBuffer.wrap(AuthenticationTokenSerializer.serialize(getToken())), instance.getInstanceID());
 +    if (getToken().isDestroyed())
 +      throw new RuntimeException("Token has been destroyed", new AccumuloSecurityException(getPrincipal(), SecurityErrorCode.TOKEN_EXPIRED));
 +    return tCreds;
 +  }
 +
 +  /**
 +   * Converts a given thrift object to our internal Credentials representation.
 +   * @param serialized a Thrift encoded set of credentials
 +   * @return a new Credentials instance; destroy the token when you're done.
 +   */
 +  public static Credentials fromThrift(TCredentials serialized) {
 +    return new Credentials(serialized.getPrincipal(), AuthenticationTokenSerializer.deserialize(serialized.getTokenClassName(), serialized.getToken()));
 +  }
 +
 +  /**
 +   * Converts the current object to a serialized form. The object returned from this contains a non-destroyable version of the {@link AuthenticationToken}, so
 +   * references to it should be tightly controlled.
 +   * 
 +   * @return serialized form of these credentials
 +   */
 +  public final String serialize() {
-     return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(Constants.UTF8))) + ":"
-         + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(Constants.UTF8))) + ":"
++    return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(UTF_8))) + ":"
++        + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(UTF_8))) + ":"
 +        + (getToken() == null ? "-" : Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(getToken())));
 +  }
 +
 +  /**
 +   * Converts the serialized form to an instance of {@link Credentials}. The original serialized form will not be affected.
 +   * 
 +   * @param serializedForm
 +   *          serialized form of credentials
 +   * @return deserialized credentials
 +   */
 +  public static final Credentials deserialize(String serializedForm) {
 +    String[] split = serializedForm.split(":", 3);
-     String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), Constants.UTF8);
-     String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), Constants.UTF8);
++    String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), UTF_8);
++    String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), UTF_8);
 +    AuthenticationToken token = null;
 +    if (!split[2].equals("-")) {
 +      byte[] tokenBytes = Base64.decodeBase64(split[2]);
 +      token = AuthenticationTokenSerializer.deserialize(tokenType, tokenBytes);
 +    }
 +    return new Credentials(principal, token);
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    return getPrincipal() == null ? 0 : getPrincipal().hashCode();
 +  }
 +
 +  @Override
 +  public boolean equals(Object obj) {
 +    if (obj == null || !(obj instanceof Credentials))
 +      return false;
 +    Credentials other = Credentials.class.cast(obj);
 +    boolean pEq = getPrincipal() == null ? (other.getPrincipal() == null) : (getPrincipal().equals(other.getPrincipal()));
 +    if (!pEq)
 +      return false;
 +    boolean tEq = getToken() == null ? (other.getToken() == null) : (getToken().equals(other.getToken()));
 +    return tEq;
 +  }
 +
 +  @Override
 +  public String toString() {
 +    return getClass().getName() + ":" + getPrincipal() + ":" + (getToken() == null ? null : getToken().getClass().getName()) + ":<hidden>";
    }
 -  
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
index be5f008,9c5782b..178de91
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
@@@ -16,27 -16,17 +16,27 @@@
   */
  package org.apache.accumulo.core.security;
  
- import java.text.ParseException;
+ import static com.google.common.base.Charsets.UTF_8;
  
- import org.apache.accumulo.core.Constants;
+ import java.text.ParseException;
  
 +/**
 + * An exception thrown when a visibility string cannot be parsed.
 + */
  public class VisibilityParseException extends ParseException {
    private static final long serialVersionUID = 1L;
    private String visibility;
 -  
 +
 +  /**
 +   * Creates a new exception.
 +   *
 +   * @param reason reason string
 +   * @param visibility visibility that could not be parsed
 +   * @param errorOffset offset into visibility where parsing failed
 +   */
    public VisibilityParseException(String reason, byte[] visibility, int errorOffset) {
      super(reason, errorOffset);
-     this.visibility = new String(visibility, Constants.UTF8);
+     this.visibility = new String(visibility, UTF_8);
    }
    
    @Override

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
index 7a23c37,f6a6a01..8771832
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
@@@ -22,8 -24,6 +24,7 @@@ import java.util.Arrays
  import java.util.Collection;
  import java.util.List;
  
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.ByteSequence;
  import org.apache.hadoop.io.Text;
  
  public class ByteBufferUtil {
@@@ -62,18 -62,6 +63,18 @@@
    }
    
    public static String toString(ByteBuffer bytes) {
-     return new String(bytes.array(), bytes.position(), bytes.remaining(), Constants.UTF8);
+     return new String(bytes.array(), bytes.position(), bytes.remaining(), UTF_8);
    }
 +  
 +  public static ByteBuffer toByteBuffers(ByteSequence bs) {
 +    if (bs == null)
 +      return null;
 +
 +    if (bs.isBackedByArray()) {
 +      return ByteBuffer.wrap(bs.getBackingArray(), bs.offset(), bs.length());
 +    } else {
 +      // TODO create more efficient impl
 +      return ByteBuffer.wrap(bs.toArray());
 +    }
 +  }
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index 9f86db3,0000000..26a8c73
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@@ -1,117 -1,0 +1,118 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.io.IOException;
 +import java.io.PrintStream;
 +
 +import jline.console.ConsoleReader;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.ClientOpts.Password;
 +import org.apache.accumulo.core.cli.ClientOpts.PasswordConverter;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.util.Base64;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class CreateToken {
 +  
 +  private static ConsoleReader reader = null;
 +  
 +  private static ConsoleReader getConsoleReader() throws IOException {
 +    if (reader == null)
 +      reader = new ConsoleReader();
 +    return reader;
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = {"-u", "--user"}, description = "Connection user")
 +    public String principal = null;
 +    
 +    @Parameter(names = "-p", converter = PasswordConverter.class, description = "Connection password")
 +    public Password password = null;
 +    
 +    @Parameter(names = "--password", converter = PasswordConverter.class, description = "Enter the connection password", password = true)
 +    public Password securePassword = null;
 +    
 +    @Parameter(names = {"-tc", "--tokenClass"}, description = "The class of the authentication token")
 +    public String tokenClassName = PasswordToken.class.getName();
 +    
 +    @Parameter(names = {"-f", "--file"}, description = "The filename to save the auth token to. Multiple tokens can be stored in the same file,"
 +        + " but only the first for each user will be recognized.")
 +    public String tokenFile = null;
 +  }
 +  
 +  public static void main(String[] args) {
 +    Opts opts = new Opts();
 +    opts.parseArgs(CreateToken.class.getName(), args);
 +    
 +    Password pass = opts.password;
 +    if (pass == null && opts.securePassword != null) {
 +      pass = opts.securePassword;
 +    }
 +    
 +    try {
 +      String principal = opts.principal;
 +      if (principal == null) {
 +        principal = getConsoleReader().readLine("Username (aka principal): ");
 +      }
 +      
 +      AuthenticationToken token = Class.forName(opts.tokenClassName).asSubclass(AuthenticationToken.class).newInstance();
 +      Properties props = new Properties();
 +      for (TokenProperty tp : token.getProperties()) {
 +        String input;
 +        if (pass != null && tp.getKey().equals("password")) {
 +          input = pass.toString();
 +        } else {
 +          if (tp.getMask()) {
 +            input = getConsoleReader().readLine(tp.getDescription() + ": ", '*');
 +          } else {
 +            input = getConsoleReader().readLine(tp.getDescription() + ": ");
 +          }
 +        }
 +        props.put(tp.getKey(), input);
 +        token.init(props);
 +      }
 +      String tokenBase64 = Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token));
 +      
 +      String tokenFile = opts.tokenFile;
 +      if (tokenFile == null) {
 +        tokenFile = getConsoleReader().readLine("File to save auth token to: ");
 +      }
 +      File tf = new File(tokenFile);
 +      if (!tf.exists()) {
 +        tf.createNewFile();
 +      }
-       PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, Constants.UTF8.name());
++      PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, UTF_8.name());
 +      String outString = principal + ":" + opts.tokenClassName + ":" + tokenBase64;
 +      out.println(outString);
 +      out.close();
 +      System.out.println("Token written to " + tokenFile + ". Remember to upload it to hdfs.");
 +    } catch (Exception e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Encoding.java
index aff8f62,9999c3c..b26b363
--- a/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.core.util;
  
- import org.apache.accumulo.core.Constants;
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import org.apache.commons.codec.binary.Base64;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.hadoop.io.Text;
  
  public class Encoding {
@@@ -36,8 -38,10 +37,8 @@@
    public static byte[] decodeBase64FileName(String node) {
      while (node.length() % 4 != 0)
        node += "=";
 -    
 -    node = node.replace('_', '/').replace('-', '+');
 -    
 +    /* decode transparently handles URLSafe encodings */
-     return Base64.decodeBase64(node.getBytes(Constants.UTF8));
+     return Base64.decodeBase64(node.getBytes(UTF_8));
    }
    
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java
index 02c8be3,3830108..05067b5
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@@ -21,7 -23,7 +23,6 @@@ import java.util.Iterator
  import java.util.List;
  import java.util.Map.Entry;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.ClientOnRequiredTable;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Scanner;
@@@ -236,8 -233,8 +237,8 @@@ public class Merge 
          while (iterator.hasNext()) {
            Entry<Key,Value> entry = iterator.next();
            Key key = entry.getKey();
 -          if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
 +          if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
-             String[] sizeEntries = new String(entry.getValue().get(), Constants.UTF8).split(",");
+             String[] sizeEntries = new String(entry.getValue().get(), UTF_8).split(",");
              if (sizeEntries.length == 2) {
                tabletSize += Long.parseLong(sizeEntries[0]);
              }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
index 1ebbf13,0000000..e71a377
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
@@@ -1,31 -1,0 +1,33 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class MonitorUtil {
 +  public static String getLocation(Instance instance) throws KeeperException, InterruptedException {
 +    ZooReader zr = new ZooReader(instance.getZooKeepers(), 5000);
 +    byte[] loc = zr.getData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, null);
-     return loc==null ? null : new String(loc, Constants.UTF8);
++    return loc==null ? null : new String(loc, UTF_8);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index da5648c,755bee2..b187a76
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -557,14 -437,11 +559,14 @@@ public class Shell extends ShellOption
      } catch (IOException e) {
        log.warn("Unable to load history file at " + historyPath);
      }
 -    
 +
 +    // Turn Ctrl+C into Exception instead of JVM exit
 +    reader.setHandleUserInterrupt(true);
 +
      ShellCompletor userCompletor = null;
 -    
 +
      if (execFile != null) {
-       java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
 -      java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++      java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
        try {
          while (scanner.hasNextLine() && !hasExited()) {
            execCommand(scanner.nextLine(), true, isVerbose());
@@@ -974,14 -805,14 +976,14 @@@
      @Override
      public void close() {}
    };
 -  
 +
    public static class PrintFile implements PrintLine {
      PrintWriter writer;
 -    
 +
      public PrintFile(String filename) throws FileNotFoundException {
-       writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), Constants.UTF8)));
+       writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), UTF_8)));
      }
 -    
 +
      @Override
      public void print(String s) {
        writer.println(s);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
index f06a639,c325612..90a3548
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
@@@ -19,9 -21,7 +21,8 @@@ import static com.google.common.base.Ch
  import java.io.File;
  import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.accumulo.core.util.shell.Shell;
  import org.apache.accumulo.core.util.shell.Shell.Command;
  import org.apache.commons.cli.CommandLine;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
index bcb6c24,5c86ac9..44dce0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
@@@ -35,12 -36,12 +36,12 @@@ public class AuthenticateCommand extend
      final String user = cl.getArgs()[0];
      final String p = shellState.readMaskedLine("Enter current password for '" + user + "': ", '*');
      if (p == null) {
 -      shellState.getReader().printNewline();
 +      shellState.getReader().println();
        return 0;
      } // user canceled
-     final byte[] password = p.getBytes(Constants.UTF8);
+     final byte[] password = p.getBytes(UTF_8);
      final boolean valid = shellState.getConnector().securityOperations().authenticateUser(user, new PasswordToken(password));
 -    shellState.getReader().printString((valid ? "V" : "Not v") + "alid\n");
 +    shellState.getReader().println((valid ? "V" : "Not v") + "alid");
      return 0;
    }
    

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
index d2c73f7,8e39115..fc1af0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
@@@ -16,16 -16,16 +16,17 @@@
   */
  package org.apache.accumulo.core.util.shell.commands;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.io.File;
  import java.io.IOException;
 +import java.util.Map;
  import java.util.Map.Entry;
  import java.util.Scanner;
 +import java.util.Set;
  import java.util.SortedSet;
  import java.util.TreeSet;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.AccumuloException;
  import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.TableExistsException;
@@@ -73,12 -70,12 +74,12 @@@ public class CreateTableCommand extend
      }
      final SortedSet<Text> partitions = new TreeSet<Text>();
      final boolean decode = cl.hasOption(base64Opt.getOpt());
 -    
 +
      if (cl.hasOption(createTableOptSplit.getOpt())) {
        final String f = cl.getOptionValue(createTableOptSplit.getOpt());
 -      
 +
        String line;
-       Scanner file = new Scanner(new File(f), Constants.UTF8.name());
+       Scanner file = new Scanner(new File(f), UTF_8.name());
        try {
          while (file.hasNextLine()) {
            line = file.nextLine();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
index 61f60f8,4f1c793..06d36b9
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
@@@ -19,8 -21,6 +21,7 @@@ import static com.google.common.base.Ch
  import java.security.SecureRandom;
  import java.util.Random;
  
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.Base64;
  import org.apache.accumulo.core.util.shell.Shell;
  import org.apache.accumulo.core.util.shell.Shell.Command;
  import org.apache.accumulo.core.util.shell.ShellCommandException;
@@@ -39,11 -40,12 +40,11 @@@ public class HiddenCommand extends Comm
    public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
      if (rand.nextInt(10) == 0) {
        shellState.getReader().beep();
 -      shellState.getReader().printNewline();
 -      shellState.getReader().printString(
 +      shellState.getReader().println();
 +      shellState.getReader().println(
            new String(Base64.decodeBase64(("ICAgICAgIC4tLS4KICAgICAgLyAvXCBcCiAgICAgKCAvLS1cICkKICAgICAuPl8gIF88LgogICAgLyB8ICd8ICcgXAog"
                + "ICAvICB8Xy58Xy4gIFwKICAvIC98ICAgICAgfFwgXAogfCB8IHwgfFwvfCB8IHwgfAogfF98IHwgfCAgfCB8IHxffAogICAgIC8gIF9fICBcCiAgICAvICAv"
-               + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(Constants.UTF8)), Constants.UTF8));
+               + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(UTF_8)), UTF_8));
 -      shellState.getReader().printNewline();
      } else {
        throw new ShellCommandException(ErrorCode.UNRECOGNIZED_COMMAND, getName());
      }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/PasswdCommand.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
index a0743f4,01fb92e..b8fc4c4
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
@@@ -41,10 -42,10 +42,10 @@@ public class UserCommand extends Comman
      // of these methods fails
      final String p = shellState.readMaskedLine("Enter password for user " + user + ": ", '*');
      if (p == null) {
 -      shellState.getReader().printNewline();
 +      shellState.getReader().println();
        return 0;
      } // user canceled
-     pass = p.getBytes(Constants.UTF8);
+     pass = p.getBytes(UTF_8);
      shellState.updateUser(user, new PasswordToken(pass));
      return 0;
    }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index 259cb06,3794f29..c276056
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@@ -16,6 -16,8 +16,7 @@@
   */
  package org.apache.accumulo.core.client;
  
+ import static com.google.common.base.Charsets.UTF_8;
 -
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertNotEquals;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
index dde5575,0000000..dce742f
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
@@@ -1,157 -1,0 +1,158 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.expect;
 +import static org.easymock.EasyMock.replay;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.List;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ZooKeeperInstanceTest {
 +  private static final UUID IID = UUID.randomUUID();
 +  private static final String IID_STRING = IID.toString();
 +  private ClientConfiguration config;
 +  private ZooCacheFactory zcf;
 +  private ZooCache zc;
 +  private ZooKeeperInstance zki;
 +
 +  private void mockIdConstruction(ClientConfiguration config) {
 +    expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
 +    expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
 +    expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
 +    expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
 +  }
 +
 +  private void mockNameConstruction(ClientConfiguration config) {
 +    expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(null);
 +    expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn("instance");
 +    expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
 +    expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
 +  }
 +
 +  @Before
 +  public void setUp() {
 +    config = createMock(ClientConfiguration.class);
 +    mockNameConstruction(config);
 +    replay(config);
 +    zcf = createMock(ZooCacheFactory.class);
 +    zc = createMock(ZooCache.class);
 +    expect(zcf.getZooCache("zk1", 30000)).andReturn(zc).anyTimes();
 +    replay(zcf);
 +    zki = new ZooKeeperInstance(config, zcf);
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void testInvalidConstruction() {
 +    config = createMock(ClientConfiguration.class);
 +    expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
 +    mockNameConstruction(config);
 +    replay(config);
 +    new ZooKeeperInstance(config);
 +  }
 +
 +  @Test
 +  public void testSimpleGetters() {
 +    assertEquals("instance", zki.getInstanceName());
 +    assertEquals("zk1", zki.getZooKeepers());
 +    assertEquals(30000, zki.getZooKeepersSessionTimeOut());
 +  }
 +
 +  @Test
 +  public void testGetInstanceID_FromCache() {
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
 +    replay(zc);
 +    assertEquals(IID_STRING, zki.getInstanceID());
 +  }
 +
 +  @Test
 +  public void testGetInstanceID_Direct() {
 +    config = createMock(ClientConfiguration.class);
 +    mockIdConstruction(config);
 +    replay(config);
 +    zki = new ZooKeeperInstance(config, zcf);
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
 +    replay(zc);
 +    assertEquals(IID_STRING, zki.getInstanceID());
 +  }
 +
 +  @Test(expected = RuntimeException.class)
 +  public void testGetInstanceID_NoMapping() {
 +    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(null);
 +    replay(zc);
 +    zki.getInstanceID();
 +  }
 +
 +  @Test(expected = RuntimeException.class)
 +  public void testGetInstanceID_IDMissingForName() {
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
 +    replay(zc);
 +    zki.getInstanceID();
 +  }
 +
 +  @Test(expected = RuntimeException.class)
 +  public void testGetInstanceID_IDMissingForID() {
 +    config = createMock(ClientConfiguration.class);
 +    mockIdConstruction(config);
 +    replay(config);
 +    zki = new ZooKeeperInstance(config, zcf);
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
 +    replay(zc);
 +    zki.getInstanceID();
 +  }
 +
 +  @Test
 +  public void testGetInstanceName() {
 +    config = createMock(ClientConfiguration.class);
 +    mockIdConstruction(config);
 +    replay(config);
 +    zki = new ZooKeeperInstance(config, zcf);
 +    expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
 +    List<String> children = new java.util.ArrayList<String>();
 +    children.add("child1");
 +    children.add("child2");
 +    expect(zc.getChildren(Constants.ZROOT + Constants.ZINSTANCES)).andReturn(children);
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(Constants.UTF8));
-     expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(UTF_8));
++    expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(UTF_8));
 +    replay(zc);
 +    assertEquals("child2", zki.getInstanceName());
 +  }
 +
 +  @Test
 +  public void testAllZooKeepersAreUsed() {
 +    final String zookeepers = "zk1,zk2,zk3", instanceName = "accumulo";
 +    ZooCacheFactory factory = createMock(ZooCacheFactory.class);
 +    expect(factory.getZooCache(zookeepers, 30000)).andReturn(zc).anyTimes();
 +    replay(factory);
 +    ClientConfiguration cfg = ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers);
 +    ZooKeeperInstance zki = new ZooKeeperInstance(cfg, factory);
 +    assertEquals(zookeepers, zki.getZooKeepers());
 +    assertEquals(instanceName, zki.getInstanceName());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index db4a12b,0000000..e2661e2
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -1,306 -1,0 +1,306 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.accumulo.core.client.mock;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertTrue;
 +import static org.junit.Assert.fail;
 +
 +import java.util.EnumSet;
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.NamespaceNotEmptyException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.NamespaceOperations;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.Filter;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.junit.Test;
 +
 +public class MockNamespacesTest {
 +
 +  Random random = new Random();
 +
 +  /**
 +   * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
 +   */
 +  @Test
 +  public void testDefaultNamespace() throws Exception {
 +    String tableName = "test";
 +    Instance instance = new MockInstance("default");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
 +    c.tableOperations().create(tableName);
 +    assertTrue(c.tableOperations().exists(tableName));
 +  }
 +
 +  /**
 +   * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2"
 +   * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the
 +   * tables and delete the namespace.
 +   */
 +  @Test
 +  public void testCreateAndDeleteNamespace() throws Exception {
 +    String namespace = "testing";
 +    String tableName1 = namespace + ".table1";
 +    String tableName2 = namespace + ".table2";
 +
 +    Instance instance = new MockInstance("createdelete");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.namespaceOperations().create(namespace);
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +
 +    c.tableOperations().create(tableName1);
 +    assertTrue(c.tableOperations().exists(tableName1));
 +
 +    c.tableOperations().create(tableName2);
 +    assertTrue(c.tableOperations().exists(tableName2));
 +
 +    // deleting
 +    try {
 +      // can't delete a namespace with tables in it
 +      c.namespaceOperations().delete(namespace);
 +      fail();
 +    } catch (NamespaceNotEmptyException e) {
 +      // ignore, supposed to happen
 +    }
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +    assertTrue(c.tableOperations().exists(tableName1));
 +    assertTrue(c.tableOperations().exists(tableName2));
 +
 +    c.tableOperations().delete(tableName2);
 +    assertTrue(!c.tableOperations().exists(tableName2));
 +    assertTrue(c.namespaceOperations().exists(namespace));
 +
 +    c.tableOperations().delete(tableName1);
 +    assertTrue(!c.tableOperations().exists(tableName1));
 +    c.namespaceOperations().delete(namespace);
 +    assertTrue(!c.namespaceOperations().exists(namespace));
 +  }
 +
 +  /**
 +   * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a
 +   * namespace-wide level, use {@link NamespaceOperations}.
 +   * 
 +   * Checks to make sure namespace-level properties are overridden by table-level properties.
 +   * 
 +   * Checks to see if the default namespace's properties work as well.
 +   */
 +
 +  @Test
 +  public void testNamespaceProperties() throws Exception {
 +    String namespace = "propchange";
 +    String tableName1 = namespace + ".table1";
 +    String tableName2 = namespace + ".table2";
 +
 +    String propKey = Property.TABLE_SCAN_MAXMEM.getKey();
 +    String propVal = "42K";
 +
 +    Instance instance = new MockInstance("props");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName1);
 +    c.namespaceOperations().setProperty(namespace, propKey, propVal);
 +
 +    // check the namespace has the property
 +    assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal));
 +
 +    // check that the table gets it from the namespace
 +    assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
 +
 +    // test a second table to be sure the first wasn't magical
 +    // (also, changed the order, the namespace has the property already)
 +    c.tableOperations().create(tableName2);
 +    assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
 +
 +    // test that table properties override namespace properties
 +    String propKey2 = Property.TABLE_FILE_MAX.getKey();
 +    String propVal2 = "42";
 +    String tablePropVal = "13";
 +
 +    c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
 +    c.namespaceOperations().setProperty("propchange", propKey2, propVal2);
 +
 +    assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
 +
 +    // now check that you can change the default namespace's properties
 +    propVal = "13K";
 +    String tableName = "some_table";
 +    c.tableOperations().create(tableName);
 +    c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal);
 +
 +    assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
 +
 +    // test the properties server-side by configuring an iterator.
 +    // should not show anything with column-family = 'a'
 +    String tableName3 = namespace + ".table3";
 +    c.tableOperations().create(tableName3);
 +
 +    IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName());
 +    c.namespaceOperations().attachIterator(namespace, setting);
 +
 +    BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig());
 +    Mutation m = new Mutation("r");
 +    m.put("a", "b", new Value("abcde".getBytes()));
 +    bw.addMutation(m);
 +    bw.flush();
 +    bw.close();
 +
 +    // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY);
 +    // do scanners work correctly in mock?
 +    // assertTrue(!s.iterator().hasNext());
 +  }
 +
 +  /**
 +   * This test renames and clones two separate table into different namespaces. different namespace.
 +   */
 +  @Test
 +  public void testRenameAndCloneTableToNewNamespace() throws Exception {
 +    String namespace1 = "renamed";
 +    String namespace2 = "cloned";
 +    String tableName = "table";
 +    String tableName1 = "renamed.table1";
 +    // String tableName2 = "cloned.table2";
 +
 +    Instance instance = new MockInstance("renameclone");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.tableOperations().create(tableName);
 +    c.namespaceOperations().create(namespace1);
 +    c.namespaceOperations().create(namespace2);
 +
 +    c.tableOperations().rename(tableName, tableName1);
 +
 +    assertTrue(c.tableOperations().exists(tableName1));
 +    assertTrue(!c.tableOperations().exists(tableName));
 +
 +    // TODO implement clone in mock
 +    /*
 +     * c.tableOperations().clone(tableName1, tableName2, false, null, null);
 +     * 
 +     * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2));
 +     */
 +    return;
 +  }
 +
 +  /**
 +   * This test renames a namespace and ensures that its tables are still correct
 +   */
 +  @Test
 +  public void testNamespaceRename() throws Exception {
 +    String namespace1 = "n1";
 +    String namespace2 = "n2";
 +    String table = "t";
 +
 +    Instance instance = new MockInstance("rename");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    c.namespaceOperations().create(namespace1);
 +    c.tableOperations().create(namespace1 + "." + table);
 +
 +    c.namespaceOperations().rename(namespace1, namespace2);
 +
 +    assertTrue(!c.namespaceOperations().exists(namespace1));
 +    assertTrue(c.namespaceOperations().exists(namespace2));
 +    assertTrue(!c.tableOperations().exists(namespace1 + "." + table));
 +    assertTrue(c.tableOperations().exists(namespace2 + "." + table));
 +  }
 +
 +  /**
 +   * This tests adding iterators to a namespace, listing them, and removing them
 +   */
 +  @Test
 +  public void testNamespaceIterators() throws Exception {
 +    Instance instance = new MockInstance("Iterators");
 +    Connector c = instance.getConnector("user", new PasswordToken("pass"));
 +
 +    String namespace = "iterator";
 +    String tableName = namespace + ".table";
 +    String iter = "thing";
 +
 +    c.namespaceOperations().create(namespace);
 +    c.tableOperations().create(tableName);
 +
 +    IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName());
 +    HashSet<IteratorScope> scope = new HashSet<IteratorScope>();
 +    scope.add(IteratorScope.scan);
 +    c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope));
 +
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    Mutation m = new Mutation("r");
-     m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
++    m.put("a", "b", new Value("abcde".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +    bw.flush();
 +
 +    Scanner s = c.createScanner(tableName, Authorizations.EMPTY);
 +    System.out.println(s.iterator().next());
 +    // do scanners work correctly in mock?
 +    // assertTrue(!s.iterator().hasNext());
 +
 +    assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter));
 +    c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope));
 +  }
 +
 +  private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
 +    for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
 +      if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
 +        return true;
 +      }
 +    }
 +    return false;
 +  }
 +
 +  private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException,
 +      AccumuloSecurityException {
 +    for (Entry<String,String> e : c.namespaceOperations().getProperties(n)) {
 +      if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
 +        return true;
 +      }
 +    }
 +    return false;
 +  }
 +
 +  public static class SimpleFilter extends Filter {
 +    @Override
 +    public boolean accept(Key k, Value v) {
 +      if (k.getColumnFamily().toString().equals("a"))
 +        return false;
 +      return true;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
index 17d2297,0000000..7bb3fa4
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
@@@ -1,129 -1,0 +1,130 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.security.tokens;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.IOException;
 +import java.net.URL;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
 +import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
 +import org.junit.Assert;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +public class CredentialProviderTokenTest {
 +
 +  private static boolean isCredentialProviderAvailable = false;
 +
 +  // Keystore contains: {'root.password':'password', 'bob.password':'bob'}
 +  private static String keystorePath;
 +
 +  @BeforeClass
 +  public static void setup() {
 +    try {
 +      Class.forName(CredentialProviderFactoryShim.HADOOP_CRED_PROVIDER_CLASS_NAME);
 +      isCredentialProviderAvailable = true;
 +    } catch (Exception e) {
 +      isCredentialProviderAvailable = false;
 +    }
 +
 +    URL keystoreUrl = CredentialProviderTokenTest.class.getResource("/passwords.jceks");
 +    Assert.assertNotNull(keystoreUrl);
 +    keystorePath = "jceks://file/" + new File(keystoreUrl.getFile()).getAbsolutePath();
 +  }
 +
 +  @Test
 +  public void testPasswordsFromCredentialProvider() throws Exception {
 +    if (!isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
-     Assert.assertArrayEquals("password".getBytes(Constants.UTF8), token.getPassword());
++    Assert.assertArrayEquals("password".getBytes(UTF_8), token.getPassword());
 +
 +    token = new CredentialProviderToken("bob.password", keystorePath);
-     Assert.assertArrayEquals("bob".getBytes(Constants.UTF8), token.getPassword());
++    Assert.assertArrayEquals("bob".getBytes(UTF_8), token.getPassword());
 +  }
 +
 +  @Test
 +  public void testEqualityAfterInit() throws Exception {
 +    if (!isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
 +
 +    CredentialProviderToken uninitializedToken = new CredentialProviderToken();
 +    Properties props = new Properties();
 +    props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
 +    props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
 +    uninitializedToken.init(props);
 +
 +    Assert.assertArrayEquals(token.getPassword(), uninitializedToken.getPassword());
 +  }
 +
 +  @Test
 +  public void testMissingClassesThrowsException() throws Exception {
 +    if (isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    try {
 +      new CredentialProviderToken("root.password", keystorePath);
 +      Assert.fail("Should fail to create CredentialProviderToken when classes are not available");
 +    } catch (IOException e) {
 +      // pass
 +    }
 +  }
 +
 +  @Test
 +  public void cloneReturnsCorrectObject() throws Exception {
 +    if (!isCredentialProviderAvailable) {
 +      return;
 +    }
 +
 +    CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
 +    CredentialProviderToken clone = token.clone();
 +
 +    Assert.assertEquals(token, clone);
 +    Assert.assertArrayEquals(token.getPassword(), clone.getPassword());
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void missingProperties() throws Exception {
 +    CredentialProviderToken token = new CredentialProviderToken();
 +    token.init(new Properties());
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void missingNameProperty() throws Exception {
 +    CredentialProviderToken token = new CredentialProviderToken();
 +    Properties props = new Properties();
 +    props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
 +    token.init(props);
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void missingProviderProperty() throws Exception {
 +    CredentialProviderToken token = new CredentialProviderToken();
 +    Properties props = new Properties();
 +    props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
 +    token.init(props);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
index 3570610,0000000..d9a47d7
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
@@@ -1,251 -1,0 +1,251 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ConditionTest {
 +  private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
 +  private static final String FAMILY = "family";
 +  private static final String QUALIFIER = "qualifier";
 +  private static final String VISIBILITY = "visibility";
 +  private static final String VALUE = "value";
 +  private static final IteratorSetting[] ITERATORS = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
 +      new IteratorSetting(3, "third", "yetanotherclass")};
 +
 +  private String toString(ByteSequence bs) {
 +    if (bs == null) {
 +      return null;
 +    }
-     return new String(bs.toArray(), Constants.UTF8);
++    return new String(bs.toArray(), UTF_8);
 +  }
 +
 +  private Condition c;
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    c = new Condition(FAMILY, QUALIFIER);
 +  }
 +
 +  @Test
 +  public void testConstruction_CharSequence() {
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteArray() {
-     c = new Condition(FAMILY.getBytes(Constants.UTF8), QUALIFIER.getBytes(Constants.UTF8));
++    c = new Condition(FAMILY.getBytes(UTF_8), QUALIFIER.getBytes(UTF_8));
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testConstruction_Text() {
 +    c = new Condition(new Text(FAMILY), new Text(QUALIFIER));
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteSequence() {
-     c = new Condition(new ArrayByteSequence(FAMILY.getBytes(Constants.UTF8)), new ArrayByteSequence(QUALIFIER.getBytes(Constants.UTF8)));
++    c = new Condition(new ArrayByteSequence(FAMILY.getBytes(UTF_8)), new ArrayByteSequence(QUALIFIER.getBytes(UTF_8)));
 +    assertEquals(FAMILY, toString(c.getFamily()));
 +    assertEquals(QUALIFIER, toString(c.getQualifier()));
 +    assertEquals(EMPTY, c.getVisibility());
 +  }
 +
 +  @Test
 +  public void testGetSetTimestamp() {
 +    c.setTimestamp(1234L);
 +    assertEquals(Long.valueOf(1234L), c.getTimestamp());
 +  }
 +
 +  @Test
 +  public void testSetValue_CharSequence() {
 +    c.setValue(VALUE);
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testSetValue_ByteArray() {
-     c.setValue(VALUE.getBytes(Constants.UTF8));
++    c.setValue(VALUE.getBytes(UTF_8));
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testSetValue_Text() {
 +    c.setValue(new Text(VALUE));
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testSetValue_ByteSequence() {
-     c.setValue(new ArrayByteSequence(VALUE.getBytes(Constants.UTF8)));
++    c.setValue(new ArrayByteSequence(VALUE.getBytes(UTF_8)));
 +    assertEquals(VALUE, toString(c.getValue()));
 +  }
 +
 +  @Test
 +  public void testGetSetVisibility() {
 +    ColumnVisibility vis = new ColumnVisibility(VISIBILITY);
 +    c.setVisibility(vis);
 +    assertEquals(VISIBILITY, toString(c.getVisibility()));
 +  }
 +
 +  @Test
 +  public void testGetSetIterators() {
 +    c.setIterators(ITERATORS);
 +    assertArrayEquals(ITERATORS, c.getIterators());
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void testSetIterators_DuplicateName() {
 +    IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
 +        new IteratorSetting(3, "first", "yetanotherclass")};
 +    c.setIterators(iterators);
 +  }
 +
 +  @Test(expected = IllegalArgumentException.class)
 +  public void testSetIterators_DuplicatePriority() {
 +    IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
 +        new IteratorSetting(1, "third", "yetanotherclass")};
 +    c.setIterators(iterators);
 +  }
 +
 +  @Test
 +  public void testEquals() {
 +    ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
 +    c.setVisibility(cvis);
 +    c.setValue(VALUE);
 +    c.setTimestamp(1234L);
 +    c.setIterators(ITERATORS);
 +
 +    // reflexivity
 +    assertTrue(c.equals(c));
 +
 +    // non-nullity
 +    assertFalse(c.equals(null));
 +
 +    // symmetry
 +    Condition c2 = new Condition(FAMILY, QUALIFIER);
 +    c2.setVisibility(cvis);
 +    c2.setValue(VALUE);
 +    c2.setTimestamp(1234L);
 +    c2.setIterators(ITERATORS);
 +    assertTrue(c.equals(c2));
 +    assertTrue(c2.equals(c));
 +
 +    Condition c3 = new Condition("nope", QUALIFIER);
 +    c3.setVisibility(cvis);
 +    c3.setValue(VALUE);
 +    c3.setTimestamp(1234L);
 +    c3.setIterators(ITERATORS);
 +    assertFalse(c.equals(c3));
 +    assertFalse(c3.equals(c));
 +    c3 = new Condition(FAMILY, "nope");
 +    c3.setVisibility(cvis);
 +    c3.setValue(VALUE);
 +    c3.setTimestamp(1234L);
 +    c3.setIterators(ITERATORS);
 +    assertFalse(c.equals(c3));
 +    assertFalse(c3.equals(c));
 +
 +    c2.setVisibility(new ColumnVisibility("sekrit"));
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setVisibility(cvis);
 +    c2.setValue(EMPTY);
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setValue(VALUE);
 +    c2.setTimestamp(2345L);
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setTimestamp(1234L);
 +    c2.setIterators(new IteratorSetting[0]);
 +    assertFalse(c.equals(c2));
 +    assertFalse(c2.equals(c));
 +    c2.setIterators(ITERATORS);
 +    assertTrue(c.equals(c2));
 +    assertTrue(c2.equals(c));
 +    
 +    // set everything but vis, so its null
 +    Condition c4 = new Condition(FAMILY, QUALIFIER);
 +    c4.setValue(VALUE);
 +    c4.setTimestamp(1234L);
 +    c4.setIterators(ITERATORS);
 +    
 +    assertFalse(c.equals(c4));
 +    assertFalse(c4.equals(c));
 +    
 +    // set everything but timestamp, so its null
 +    Condition c5 = new Condition(FAMILY, QUALIFIER);
 +    c5.setVisibility(cvis);
 +    c5.setValue(VALUE);
 +    c5.setIterators(ITERATORS);
 +    
 +    assertFalse(c.equals(c5));
 +    assertFalse(c5.equals(c));
 +    
 +    // set everything but value
 +    Condition c6 = new Condition(FAMILY, QUALIFIER);
 +    c6.setVisibility(cvis);
 +    c6.setTimestamp(1234L);
 +    c6.setIterators(ITERATORS);
 +    
 +    assertFalse(c.equals(c6));
 +    assertFalse(c6.equals(c));
 +    
 +    // test w/ no optional fields set
 +    Condition c7 = new Condition(FAMILY, QUALIFIER);
 +    Condition c8 = new Condition(FAMILY, QUALIFIER);
 +    assertTrue(c7.equals(c8));
 +    assertTrue(c8.equals(c7));
 +
 +  }
 +
 +  @Test
 +  public void testHashCode() {
 +    ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
 +    c.setVisibility(cvis);
 +    c.setValue(VALUE);
 +    c.setTimestamp(1234L);
 +    c.setIterators(ITERATORS);
 +    int hc1 = c.hashCode();
 +
 +    Condition c2 = new Condition(FAMILY, QUALIFIER);
 +    c2.setVisibility(cvis);
 +    c2.setValue(VALUE);
 +    c2.setTimestamp(1234L);
 +    c2.setIterators(ITERATORS);
 +    assertTrue(c.equals(c2));
 +    assertEquals(hc1, c2.hashCode());
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 47f44f1,0000000..908c391
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@@ -1,149 -1,0 +1,149 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.data;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertArrayEquals;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +public class ConditionalMutationTest {
-   private static final byte[] ROW = "row".getBytes(Constants.UTF8);
++  private static final byte[] ROW = "row".getBytes(UTF_8);
 +  private static final String FAMILY = "family";
 +  private static final String QUALIFIER = "qualifier";
 +  private static final String QUALIFIER2 = "qualifier2";
 +  private static final String QUALIFIER3 = "qualifier3";
 +
 +  private Condition c1, c2;
 +  private ConditionalMutation cm;
 +
 +  @Before
 +  public void setUp() throws Exception {
 +    c1 = new Condition(FAMILY, QUALIFIER);
 +    c2 = new Condition(FAMILY, QUALIFIER2);
 +    assertFalse(c1.equals(c2));
 +    cm = new ConditionalMutation(ROW, c1, c2);
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteArray() {
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteArray_StartAndLength() {
 +    cm = new ConditionalMutation(ROW, 1, 1, c1, c2);
-     assertArrayEquals("o".getBytes(Constants.UTF8), cm.getRow());
++    assertArrayEquals("o".getBytes(UTF_8), cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_Text() {
 +    cm = new ConditionalMutation(new Text(ROW), c1, c2);
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_CharSequence() {
-     cm = new ConditionalMutation(new String(ROW, Constants.UTF8), c1, c2);
++    cm = new ConditionalMutation(new String(ROW, UTF_8), c1, c2);
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testConstruction_ByteSequence() {
 +    cm = new ConditionalMutation(new ArrayByteSequence(ROW), c1, c2);
 +    assertArrayEquals(ROW, cm.getRow());
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(2, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +  }
 +
 +  @Test
 +  public void testCopyConstructor() {
 +    ConditionalMutation cm2 = new ConditionalMutation(cm);
 +    assertArrayEquals(cm.getRow(), cm2.getRow());
 +    assertEquals(cm.getConditions(), cm2.getConditions());
 +  }
 +
 +  @Test
 +  public void testAddCondition() {
 +    Condition c3 = new Condition(FAMILY, QUALIFIER3);
 +    cm.addCondition(c3);
 +    List<Condition> cs = cm.getConditions();
 +    assertEquals(3, cs.size());
 +    assertEquals(c1, cs.get(0));
 +    assertEquals(c2, cs.get(1));
 +    assertEquals(c3, cs.get(2));
 +  }
 +
 +  @Test
 +  public void testEquals() {
 +    // reflexivity
 +    assertTrue(cm.equals(cm));
 +
 +    // non-nullity
 +    assertFalse(cm.equals((Object) null));
 +
 +    // symmetry
 +    ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
 +    assertTrue(cm.equals(cm2));
 +    assertTrue(cm2.equals(cm));
 +
-     ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(Constants.UTF8), c1, c2);
++    ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(UTF_8), c1, c2);
 +    assertFalse(cm.equals(cm3));
 +    cm3 = new ConditionalMutation(ROW, c2, c1);
 +    assertFalse(cm.getConditions().equals(cm3.getConditions()));
 +    assertFalse(cm.equals(cm3));
 +  }
 +
 +  @Test
 +  public void testEquals_Mutation() {
 +    Mutation m = new Mutation(ROW);
 +    assertFalse(m.equals(cm));
 +    assertFalse(cm.equals(m));
 +  }
 +
 +  @Test
 +  public void testHashcode() {
 +    ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
 +    assertTrue(cm.equals(cm2));
 +    assertEquals(cm2.hashCode(), cm.hashCode());
 +  }
 +}


[21/50] [abbrv] git commit: Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
Merge branch '1.5' into 1.6


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9b20a9d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9b20a9d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9b20a9d4

Branch: refs/heads/master
Commit: 9b20a9d4062f9ac0d34ce4b2acd7b9b72524a84f
Parents: 5e0f6a4 2deabd3
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Oct 31 23:37:27 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Oct 31 23:37:27 2014 -0400

----------------------------------------------------------------------
 assemble/bin/bootstrap_config.sh                |  2 +-
 .../org/apache/accumulo/core/Constants.java     | 13 +++++-
 .../apache/accumulo/core/cli/ClientOpts.java    |  6 ++-
 .../accumulo/core/client/BatchWriterConfig.java | 11 ++---
 .../accumulo/core/client/ZooKeeperInstance.java | 14 ++++---
 .../client/impl/InstanceOperationsImpl.java     |  4 +-
 .../accumulo/core/client/impl/Namespaces.java   |  6 ++-
 .../accumulo/core/client/impl/ServerClient.java |  4 +-
 .../core/client/impl/TableOperationsImpl.java   | 32 ++++++++-------
 .../accumulo/core/client/impl/Tables.java       | 10 +++--
 .../accumulo/core/client/impl/Writer.java       |  5 ++-
 .../core/client/lexicoder/StringLexicoder.java  |  6 +--
 .../core/client/mapreduce/RangeInputSplit.java  |  7 ++--
 .../mapreduce/lib/impl/ConfiguratorBase.java    |  4 +-
 .../mapreduce/lib/impl/InputConfigurator.java   |  8 ++--
 .../mapreduce/lib/impl/OutputConfigurator.java  |  7 ++--
 .../lib/partition/RangePartitioner.java         |  7 ++--
 .../accumulo/core/client/mock/MockShell.java    | 11 ++---
 .../client/security/tokens/PasswordToken.java   |  5 ++-
 .../accumulo/core/conf/ConfigurationDocGen.java |  9 ++--
 .../accumulo/core/data/ArrayByteSequence.java   |  8 ++--
 .../org/apache/accumulo/core/data/Column.java   |  6 +--
 .../apache/accumulo/core/data/Condition.java    |  9 ++--
 .../org/apache/accumulo/core/data/Value.java    |  4 +-
 .../accumulo/core/file/BloomFilterLayer.java    |  7 ++--
 .../core/iterators/ColumnFamilyCounter.java     |  5 ++-
 .../accumulo/core/iterators/LongCombiner.java   |  7 ++--
 .../accumulo/core/iterators/conf/ColumnSet.java |  7 ++--
 .../core/iterators/user/BigDecimalCombiner.java |  7 ++--
 .../core/iterators/user/GrepIterator.java       |  5 ++-
 .../iterators/user/IntersectingIterator.java    |  7 ++--
 .../core/iterators/user/LargeRowFilter.java     |  5 ++-
 .../core/iterators/user/RegExFilter.java        |  7 ++--
 .../iterators/user/RowDeletingIterator.java     |  5 ++-
 .../iterators/user/SummingArrayCombiner.java    |  7 ++--
 .../iterators/user/TransformingIterator.java    |  7 ++--
 .../core/iterators/user/VisibilityFilter.java   |  5 ++-
 .../accumulo/core/security/Authorizations.java  | 13 +++---
 .../core/security/ColumnVisibility.java         | 43 ++++++++++----------
 .../accumulo/core/security/Credentials.java     | 11 ++---
 .../core/security/VisibilityConstraint.java     |  5 ++-
 .../core/security/VisibilityParseException.java |  6 +--
 .../core/trace/InstanceUserPassword.java        |  5 ++-
 .../accumulo/core/trace/ZooTraceClient.java     |  5 ++-
 .../core/util/ByteArrayBackedCharSequence.java  |  5 ++-
 .../apache/accumulo/core/util/ByteArraySet.java |  6 +--
 .../accumulo/core/util/ByteBufferUtil.java      |  5 ++-
 .../apache/accumulo/core/util/CreateToken.java  |  5 ++-
 .../org/apache/accumulo/core/util/Encoding.java |  5 ++-
 .../apache/accumulo/core/util/FastFormat.java   |  4 +-
 .../org/apache/accumulo/core/util/Merge.java    |  5 ++-
 .../apache/accumulo/core/util/MonitorUtil.java  |  4 +-
 .../org/apache/accumulo/core/util/TextUtil.java |  4 +-
 .../apache/accumulo/core/util/shell/Shell.java  |  6 ++-
 .../util/shell/commands/AddSplitsCommand.java   |  7 ++--
 .../shell/commands/AuthenticateCommand.java     |  5 ++-
 .../util/shell/commands/CreateTableCommand.java |  7 ++--
 .../util/shell/commands/ExecfileCommand.java    |  5 ++-
 .../core/util/shell/commands/HiddenCommand.java |  5 ++-
 .../core/util/shell/commands/PasswdCommand.java |  5 ++-
 .../shell/commands/QuotedStringTokenizer.java   |  5 ++-
 .../core/util/shell/commands/UserCommand.java   |  5 ++-
 .../core/client/BatchWriterConfigTest.java      |  6 +--
 .../core/client/ZooKeeperInstanceTest.java      |  9 ++--
 .../core/client/mock/MockNamespacesTest.java    |  4 +-
 .../tokens/CredentialProviderTokenTest.java     |  7 ++--
 .../security/tokens/PasswordTokenTest.java      |  7 ++--
 .../accumulo/core/data/ConditionTest.java       | 12 +++---
 .../core/data/ConditionalMutationTest.java      | 10 ++---
 .../apache/accumulo/core/data/ValueTest.java    |  5 ++-
 .../core/iterators/user/RegExFilterTest.java    |  7 ++--
 .../core/security/ColumnVisibilityTest.java     |  6 +--
 .../accumulo/core/security/CredentialsTest.java |  6 +--
 .../core/security/VisibilityConstraintTest.java |  4 +-
 .../security/crypto/BlockedIOStreamTest.java    | 10 ++---
 .../core/util/format/DeleterFormatterTest.java  | 12 +++---
 .../ShardedTableDistributionFormatterTest.java  |  8 ++--
 .../org/apache/accumulo/fate/AdminUtil.java     |  8 ++--
 .../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++----
 .../zookeeper/DistributedReadWriteLock.java     | 18 ++++----
 .../accumulo/fate/zookeeper/ZooCache.java       |  8 ++--
 .../apache/accumulo/fate/zookeeper/ZooLock.java |  6 +--
 .../accumulo/fate/zookeeper/ZooReservation.java |  9 ++--
 .../accumulo/fate/zookeeper/ZooSession.java     |  6 +--
 .../impl/MiniAccumuloClusterImpl.java           |  4 +-
 .../org/apache/accumulo/proxy/ProxyServer.java  |  7 ++--
 .../apache/accumulo/proxy/TestProxyClient.java  | 15 +++----
 .../java/org/apache/accumulo/proxy/Util.java    |  5 ++-
 .../org/apache/accumulo/server/Accumulo.java    |  4 +-
 .../accumulo/server/client/HdfsZooInstance.java | 10 +++--
 .../server/conf/ZooCachePropertyAccessor.java   |  5 ++-
 .../accumulo/server/conf/ZooConfiguration.java  |  4 +-
 .../server/constraints/MetadataConstraints.java | 14 ++++---
 .../apache/accumulo/server/init/Initialize.java | 15 +++----
 .../accumulo/server/master/LiveTServerSet.java  |  3 +-
 .../server/master/state/DeadServerList.java     |  7 ++--
 .../server/master/state/TServerInstance.java    |  7 ++--
 .../master/state/TabletStateChangeIterator.java |  5 ++-
 .../accumulo/server/master/state/ZooStore.java  |  5 ++-
 .../master/state/ZooTabletStateStore.java       | 11 ++---
 .../server/metrics/AbstractMetricsImpl.java     |  5 ++-
 .../accumulo/server/monitor/LogService.java     |  4 +-
 .../server/security/SecurityOperation.java      |  4 +-
 .../server/security/SystemCredentials.java      | 10 +++--
 .../security/handler/ZKAuthenticator.java       |  4 +-
 .../server/security/handler/ZKAuthorizor.java   |  5 ++-
 .../server/security/handler/ZKPermHandler.java  |  5 ++-
 .../accumulo/server/tables/TableManager.java    | 18 ++++----
 .../server/tablets/UniqueNameAllocator.java     | 10 +++--
 .../server/util/AddFilesWithMissingEntries.java |  5 ++-
 .../accumulo/server/util/ChangeSecret.java      |  5 ++-
 .../accumulo/server/util/CleanZookeeper.java    |  6 ++-
 .../accumulo/server/util/DeleteZooInstance.java |  6 ++-
 .../accumulo/server/util/DumpZookeeper.java     |  5 ++-
 .../accumulo/server/util/ListInstances.java     |  6 ++-
 .../accumulo/server/util/MetadataTableUtil.java | 32 ++++++++-------
 .../accumulo/server/util/NamespacePropUtil.java |  4 +-
 .../accumulo/server/util/RandomizeVolumes.java  |  6 +--
 .../accumulo/server/util/RestoreZookeeper.java  |  7 ++--
 .../accumulo/server/util/SendLogToChainsaw.java |  9 ++--
 .../accumulo/server/util/SystemPropUtil.java    |  4 +-
 .../accumulo/server/util/TablePropUtil.java     |  4 +-
 .../accumulo/server/util/TabletServerLocks.java |  4 +-
 .../server/watcher/MonitorLog4jWatcher.java     |  4 +-
 .../accumulo/server/zookeeper/ZooQueueLock.java |  9 ++--
 .../server/zookeeper/ZooReaderWriter.java       |  5 ++-
 .../zookeeper/ZooReaderWriterFactory.java       |  5 ++-
 .../server/conf/NamespaceConfigurationTest.java | 10 +++--
 .../conf/ServerConfigurationFactoryTest.java    |  4 +-
 .../server/conf/TableConfigurationTest.java     |  9 ++--
 .../conf/ZooCachePropertyAccessorTest.java      |  4 +-
 .../java/org/apache/accumulo/master/Master.java |  8 ++--
 .../master/recovery/RecoveryManager.java        |  4 +-
 .../accumulo/master/state/SetGoalState.java     |  4 +-
 .../accumulo/master/tableOps/BulkImport.java    |  8 ++--
 .../master/tableOps/CancelCompactions.java      | 10 +++--
 .../accumulo/master/tableOps/CompactRange.java  | 14 ++++---
 .../accumulo/master/tableOps/ExportTable.java   |  8 ++--
 .../accumulo/master/tableOps/ImportTable.java   | 16 ++++----
 .../accumulo/master/tableOps/RenameTable.java   |  6 ++-
 .../apache/accumulo/master/tableOps/Utils.java  | 14 ++++---
 .../master/tserverOps/ShutdownTServer.java      |  4 +-
 .../org/apache/accumulo/monitor/Monitor.java    |  6 ++-
 .../accumulo/monitor/ZooKeeperStatus.java       |  7 ++--
 .../accumulo/monitor/servlets/BasicServlet.java | 10 +++--
 .../monitor/servlets/DefaultServlet.java        |  5 ++-
 .../accumulo/monitor/servlets/ShellServlet.java |  5 ++-
 .../accumulo/monitor/servlets/XMLServlet.java   |  7 ++--
 .../accumulo/monitor/servlets/trace/Basic.java  |  5 ++-
 .../org/apache/accumulo/tracer/TraceServer.java |  8 ++--
 .../tserver/BulkFailedCopyProcessor.java        |  5 ++-
 .../org/apache/accumulo/tserver/Tablet.java     | 10 +++--
 .../apache/accumulo/tserver/TabletServer.java   |  9 ++--
 .../apache/accumulo/tserver/log/DfsLogger.java  |  4 +-
 .../accumulo/tserver/logger/LogFileValue.java   | 11 ++---
 .../accumulo/tserver/logger/LogReader.java      |  5 ++-
 .../accumulo/test/BulkImportDirectory.java      |  5 ++-
 .../accumulo/test/NativeMapConcurrencyTest.java |  5 ++-
 .../accumulo/test/NativeMapPerformanceTest.java |  7 ++--
 .../accumulo/test/NativeMapStressTest.java      |  5 ++-
 .../apache/accumulo/test/TestBinaryRows.java    |  7 ++--
 .../org/apache/accumulo/test/TestIngest.java    |  7 ++--
 .../accumulo/test/TestMultiTableIngest.java     |  5 ++-
 .../test/continuous/ContinuousIngest.java       |  8 ++--
 .../test/continuous/ContinuousMoru.java         |  7 ++--
 .../test/continuous/ContinuousQuery.java        |  5 ++-
 .../test/continuous/ContinuousScanner.java      |  5 ++-
 .../test/continuous/ContinuousVerify.java       |  7 ++--
 .../test/continuous/ContinuousWalk.java         | 17 ++++----
 .../accumulo/test/continuous/Histogram.java     |  6 +--
 .../test/continuous/PrintScanTimeHistogram.java |  5 ++-
 .../accumulo/test/continuous/TimeBinner.java    |  5 ++-
 .../test/continuous/UndefinedAnalyzer.java      |  9 ++--
 .../test/functional/CacheTestReader.java        |  7 ++--
 .../test/functional/CacheTestWriter.java        | 11 ++---
 .../accumulo/test/functional/ZombieTServer.java |  4 +-
 .../metadata/MetadataBatchScanTest.java         |  7 ++--
 .../apache/accumulo/test/randomwalk/Module.java |  7 ++--
 .../test/randomwalk/bulk/BulkMinusOne.java      |  5 ++-
 .../accumulo/test/randomwalk/bulk/Verify.java   |  5 ++-
 .../test/randomwalk/concurrent/BatchWrite.java  |  5 ++-
 .../test/randomwalk/concurrent/BulkImport.java  |  5 ++-
 .../concurrent/ChangeAuthorizations.java        |  7 ++--
 .../randomwalk/concurrent/StopTabletServer.java |  4 +-
 .../accumulo/test/randomwalk/image/Verify.java  |  5 ++-
 .../accumulo/test/randomwalk/image/Write.java   |  9 ++--
 .../test/randomwalk/multitable/Write.java       |  5 ++-
 .../randomwalk/security/SecurityHelper.java     |  7 ++--
 .../test/randomwalk/security/TableOp.java       |  9 ++--
 .../test/randomwalk/sequential/Write.java       |  5 ++-
 .../test/randomwalk/shard/BulkInsert.java       |  5 ++-
 .../test/randomwalk/shard/ExportIndex.java      |  5 ++-
 .../accumulo/test/scalability/Ingest.java       |  5 ++-
 .../apache/accumulo/test/AuditMessageIT.java    |  6 +--
 .../org/apache/accumulo/test/NamespacesIT.java  |  8 ++--
 .../java/org/apache/accumulo/test/VolumeIT.java |  6 +--
 .../accumulo/test/functional/AddSplitIT.java    |  5 ++-
 .../test/functional/BadIteratorMincIT.java      |  6 +--
 .../test/functional/BatchScanSplitIT.java       |  7 ++--
 .../test/functional/BatchWriterFlushIT.java     |  5 ++-
 .../accumulo/test/functional/BulkFileIT.java    |  5 ++-
 .../accumulo/test/functional/ConcurrencyIT.java |  7 ++--
 .../accumulo/test/functional/ConstraintIT.java  | 23 ++++++-----
 .../test/functional/CreateAndUseIT.java         |  5 ++-
 .../test/functional/DeleteEverythingIT.java     |  4 +-
 .../test/functional/DeleteRowsSplitIT.java      |  6 +--
 .../accumulo/test/functional/LargeRowIT.java    |  7 ++--
 .../accumulo/test/functional/NativeMapIT.java   | 15 +++----
 .../test/functional/ScanIteratorIT.java         |  7 ++--
 .../accumulo/test/functional/ScanRangeIT.java   |  5 ++-
 .../test/functional/ScanSessionTimeOutIT.java   |  5 ++-
 .../test/functional/SplitRecoveryIT.java        |  3 +-
 .../accumulo/test/functional/TabletIT.java      |  4 +-
 .../accumulo/test/functional/VisibilityIT.java  | 11 ++---
 .../instrument/receivers/ZooSpanClient.java     |  4 +-
 215 files changed, 900 insertions(+), 667 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/assemble/bin/bootstrap_config.sh
----------------------------------------------------------------------
diff --cc assemble/bin/bootstrap_config.sh
index fb643da,0000000..fd262f0
mode 100755,000000..100755
--- a/assemble/bin/bootstrap_config.sh
+++ b/assemble/bin/bootstrap_config.sh
@@@ -1,323 -1,0 +1,323 @@@
 +#! /usr/bin/env bash
 +
 +# Licensed to the Apache Software Foundation (ASF) under one or more
 +# contributor license agreements.  See the NOTICE file distributed with
 +# this work for additional information regarding copyright ownership.
 +# The ASF licenses this file to You under the Apache License, Version 2.0
 +# (the "License"); you may not use this file except in compliance with
 +# the License.  You may obtain a copy of the License at
 +#
 +#     http://www.apache.org/licenses/LICENSE-2.0
 +#
 +# Unless required by applicable law or agreed to in writing, software
 +# distributed under the License is distributed on an "AS IS" BASIS,
 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 +# See the License for the specific language governing permissions and
 +# limitations under the License.
 +
 +function usage {
 +  cat <<EOF
 +Usage: bootstrap_config.sh [-options]
 +where options include (long options not available on all platforms):
 +    -d, --dir        Alternate directory to setup config files
 +    -s, --size       Supported sizes: '1GB' '2GB' '3GB' '512MB'
 +    -n, --native     Configure to use native libraries
 +    -j, --jvm        Configure to use the jvm
 +    -o, --overwrite  Overwrite the default config directory
 +    -v, --version    Specify the Apache Hadoop version supported versions: '1' '2'
 +    -h, --help       Print this help message
 +EOF
 +}
 +
 +# Start: Resolve Script Directory
 +SOURCE="${BASH_SOURCE[0]}"
 +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
 +  bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 +  SOURCE="$(readlink "$SOURCE")"
 +  [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 +done
 +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 +
 +# Stop: Resolve Script Directory
 +
 +#
 +# Resolve accumulo home for bootstrapping
 +#
 +ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd )
 +TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates"
 +CONF_DIR="${ACCUMULO_HOME}/conf"
 +ACCUMULO_SITE=accumulo-site.xml
 +ACCUMULO_ENV=accumulo-env.sh
 +
 +SIZE=
 +TYPE=
 +HADOOP_VERSION=
 +OVERWRITE="0"
 +BASE_DIR=
 +
 +#Execute getopt
 +if [[ $(uname -s) == "Linux" ]]; then
 +  args=$(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@")
 +else # Darwin, BSD
 +  args=$(getopt b:d:s:njov:h $*)
 +fi
 +
 +#Bad arguments
 +if [[ $? != 0 ]]; then
 +  usage 1>&2
 +  exit 1
 +fi
 +eval set -- $args
 +
 +for i
 +do
 +  case "$i" in
 +    -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers
 +      BASE_DIR=$2; shift
 +      shift;;
 +    -d|--dir)
 +      CONF_DIR=$2; shift
 +      shift;;
 +    -s|--size)
 +      SIZE=$2; shift
 +      shift;;
 +    -n|--native)
 +      TYPE=native
 +      shift;;
 +    -j|--jvm)
 +      TYPE=jvm
 +      shift;;
 +    -o|--overwrite)
 +      OVERWRITE=1
 +      shift;;
 +    -v|--version)
 +      HADOOP_VERSION=$2; shift
 +      shift;;
 +    -h|--help)
 +      usage
 +      exit 0
 +      shift;;
 +    --)
 +    shift
 +    break;;
 +  esac
 +done
 +
 +while [[ "${OVERWRITE}" = "0" ]]; do
 +  if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then
 +    echo "Warning your current config files in ${CONF_DIR} will be overwritten!"
 +    echo
 +    echo "How would you like to proceed?:"
 +    select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do
 +      if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then
 +        echo -n "Please specifiy new conf directory: "
 +        read CONF_DIR
 +      elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then
 +        OVERWRITE=1
 +      fi
 +      break
 +    done
 +  else
 +    OVERWRITE=1
 +  fi
 +done
- echo "Coppying configuration files to: ${CONF_DIR}"
++echo "Copying configuration files to: ${CONF_DIR}"
 +
 +#Native 1GB
 +native_1GB_tServer="-Xmx128m -Xms128m"
 +_1GB_master="-Xmx128m -Xms128m"
 +_1GB_monitor="-Xmx64m -Xms64m"
 +_1GB_gc="-Xmx64m -Xms64m"
 +_1GB_other="-Xmx128m -Xms64m"
 +
 +_1GB_memoryMapMax="256M"
 +native_1GB_nativeEnabled="true"
 +_1GB_cacheDataSize="15M"
 +_1GB_cacheIndexSize="40M"
 +_1GB_sortBufferSize="50M"
 +_1GB_waLogMaxSize="256M"
 +
 +#Native 2GB
 +native_2GB_tServer="-Xmx256m -Xms256m"
 +_2GB_master="-Xmx256m -Xms256m"
 +_2GB_monitor="-Xmx128m -Xms64m"
 +_2GB_gc="-Xmx128m -Xms128m"
 +_2GB_other="-Xmx256m -Xms64m"
 +
 +_2GB_memoryMapMax="512M"
 +native_2GB_nativeEnabled="true"
 +_2GB_cacheDataSize="30M"
 +_2GB_cacheIndexSize="80M"
 +_2GB_sortBufferSize="50M"
 +_2GB_waLogMaxSize="512M"
 +
 +#Native 3GB
 +native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m"
 +_3GB_master="-Xmx1g -Xms1g"
 +_3GB_monitor="-Xmx1g -Xms256m"
 +_3GB_gc="-Xmx256m -Xms256m"
 +_3GB_other="-Xmx1g -Xms256m"
 +
 +_3GB_memoryMapMax="1G"
 +native_3GB_nativeEnabled="true"
 +_3GB_cacheDataSize="128M"
 +_3GB_cacheIndexSize="128M"
 +_3GB_sortBufferSize="200M"
 +_3GB_waLogMaxSize="1G"
 +
 +#Native 512MB
 +native_512MB_tServer="-Xmx48m -Xms48m"
 +_512MB_master="-Xmx128m -Xms128m"
 +_512MB_monitor="-Xmx64m -Xms64m"
 +_512MB_gc="-Xmx64m -Xms64m"
 +_512MB_other="-Xmx128m -Xms64m"
 +
 +_512MB_memoryMapMax="80M"
 +native_512MB_nativeEnabled="true"
 +_512MB_cacheDataSize="7M"
 +_512MB_cacheIndexSize="20M"
 +_512MB_sortBufferSize="50M"
 +_512MB_waLogMaxSize="100M"
 +
 +#JVM 1GB
 +jvm_1GB_tServer="-Xmx384m -Xms384m"
 +
 +jvm_1GB_nativeEnabled="false"
 +
 +#JVM 2GB
 +jvm_2GB_tServer="-Xmx768m -Xms768m"
 +
 +jvm_2GB_nativeEnabled="false"
 +
 +#JVM 3GB
 +jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G"
 +
 +jvm_3GB_nativeEnabled="false"
 +
 +#JVM 512MB
 +jvm_512MB_tServer="-Xmx128m -Xms128m"
 +
 +jvm_512MB_nativeEnabled="false"
 +
 +
 +if [[ -z "${SIZE}" ]]; then
 +  echo "Choose the heap configuration:"
 +  select DIRNAME in 1GB 2GB 3GB 512MB; do
 +    echo "Using '${DIRNAME}' configuration"
 +    SIZE=${DIRNAME}
 +    break
 +  done
 +elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB"  && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then
 +  echo "Invalid memory size"
 +  echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'"
 +  exit 1
 +fi
 +
 +if [[ -z "${TYPE}" ]]; then
 +  echo
 +  echo "Choose the Accumulo memory-map type:"
 +  select TYPENAME in Java Native; do
 +    if [[ "${TYPENAME}" == "Native" ]]; then
 +      TYPE="native"
 +      echo "Don't forget to build the native libraries using the bin/build_native_library.sh script"
 +    elif [[ "${TYPENAME}" == "Java" ]]; then
 +      TYPE="jvm"
 +    fi
 +    echo "Using '${TYPE}' configuration"
 +    echo
 +    break
 +  done
 +fi
 +
 +if [[ -z "${HADOOP_VERSION}" ]]; then
 +  echo
 +  echo "Choose the Apache Hadoop version:"
 +  select HADOOP in 'HADOOP 1' 'HADOOP 2' ; do
 +    if [ "${HADOOP}" == "HADOOP 2" ]; then
 +      HADOOP_VERSION="2"
 +    elif [ "${HADOOP}" == "HADOOP 1" ]; then
 +      HADOOP_VERSION="1"
 +    fi
 +    echo "Using Apache Hadoop version '${HADOOP_VERSION}' configuration"
 +    echo
 +    break
 +  done
 +elif [[ "${HADOOP_VERSION}" != "1" && "${HADOOP_VERSION}" != "2" ]]; then
 +  echo "Invalid Apache Hadoop version"
 +  echo "Supported Apache Hadoop versions: '1' '2'"
 +  exit 1
 +fi
 +
 +for var in SIZE TYPE HADOOP_VERSION; do
 +  if [[ -z ${!var} ]]; then
 +    echo "Invalid $var configuration"
 +    exit 1
 +  fi
 +done
 +
 +TSERVER="${TYPE}_${SIZE}_tServer"
 +MASTER="_${SIZE}_master"
 +MONITOR="_${SIZE}_monitor"
 +GC="_${SIZE}_gc"
 +OTHER="_${SIZE}_other"
 +
 +MEMORY_MAP_MAX="_${SIZE}_memoryMapMax"
 +NATIVE="${TYPE}_${SIZE}_nativeEnabled"
 +CACHE_DATA_SIZE="_${SIZE}_cacheDataSize"
 +CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize"
 +SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize"
 +WAL_MAX_SIZE="_${SIZE}_waLogMaxSize"
 +
 +MAVEN_PROJ_BASEDIR=""
 +
 +if [[ ! -z "${BASE_DIR}" ]]; then
 +  MAVEN_PROJ_BASEDIR="\n  <property>\n    <name>general.maven.project.basedir</name>\n    <value>${BASE_DIR}</value>\n  </property>\n"
 +fi
 +
 +#Configure accumulo-env.sh
 +mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/
 +sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \
 +    -e "s/\${masterHigh_masterLow}/${!MASTER}/" \
 +    -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \
 +    -e "s/\${gcHigh_gcLow}/${!GC}/" \
 +    -e "s/\${otherHigh_otherLow}/${!OTHER}/" \
 +    ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV
 +
 +#Configure accumulo-site.xml
 +sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \
 +    -e "s/\${nativeEnabled}/${!NATIVE}/" \
 +    -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \
 +    -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \
 +    -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \
 +    -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \
 +    -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE
 +
 +#Configure for hadoop 1
 +if [[ "$HADOOP_VERSION" = "1" ]]; then
 +  sed -e 's/^test -z \"$HADOOP_CONF_DIR\"/#test -z \"$HADOOP_CONF_DIR\"/' -e 's/^# test -z "$HADOOP_CONF_DIR"/test -z \"$HADOOP_CONF_DIR\"/' ${CONF_DIR}/$ACCUMULO_ENV > temp
 +  mv temp ${CONF_DIR}/$ACCUMULO_ENV
 +  sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements -->\n      <!--/' \
 +      -e 's/<!-- End Hadoop 2 requirements -->/-->\n      <!-- End Hadoop 2 requirements -->/' \
 +      ${CONF_DIR}/$ACCUMULO_SITE > temp
 +  mv temp  ${CONF_DIR}/$ACCUMULO_SITE
 +fi
 +
 +#Additional setup steps for native configuration.
 +if [[ "${TYPE}" = "native" ]]; then
 +  if [[ "$(uname)" = 'Linux' ]]; then
 +    if [[ -z $HADOOP_PREFIX ]]; then
 +      echo "HADOOP_PREFIX not set cannot automatically configure LD_LIBRARY_PATH"
 +    else
 +      NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null)
 +      if [[ -z $NATIVE_LIB ]]; then
 +        echo -e "Native libraries could not be found for your sytem in: $HADOOP_PREFIX"
 +      else
 +        sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp
 +        mv temp ${CONF_DIR}/$ACCUMULO_ENV
 +        echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH"
 +      fi
 +    fi
 +  fi
 +  echo -e "Please remember to compile the native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed."
 +fi
 +echo "Setup complete"

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 2a570f4,240de73..9897262
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -16,17 -16,37 +16,26 @@@
   */
  package org.apache.accumulo.core;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.nio.charset.Charset;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
  
 -import org.apache.accumulo.core.conf.AccumuloConfiguration;
 -import org.apache.accumulo.core.conf.Property;
 -import org.apache.accumulo.core.data.Key;
 -import org.apache.accumulo.core.data.KeyExtent;
 -import org.apache.accumulo.core.data.PartialKey;
 -import org.apache.accumulo.core.data.Range;
  import org.apache.accumulo.core.security.Authorizations;
 -import org.apache.accumulo.core.util.ColumnFQ;
 -import org.apache.hadoop.fs.Path;
 -import org.apache.hadoop.io.Text;
  
+ import com.google.common.base.Charsets;
+ 
  public class Constants {
-   public static final Charset UTF8 = Charset.forName("UTF-8");
+   /**
 -   * @deprecated since 1.5.3; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
++   * @deprecated since 1.5.3, 1.6.2; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
+    *             {@link Charset#forName(String)} with "UTF-8" instead
+    */
+   @Deprecated
+   public static final Charset UTF8 = UTF_8;
    public static final String VERSION = FilteredConstants.VERSION;
 -  
 -  // versions should never be negative
 -  public static final Integer WIRE_VERSION = 2;
 -  public static final int DATA_VERSION = 5;
 -  public static final int PREV_DATA_VERSION = 4;
 -  
 +
    // Zookeeper locations
    public static final String ZROOT = "/accumulo";
    public static final String ZINSTANCES = "/instances";
@@@ -56,62 -76,117 +65,62 @@@
    public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
    public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
    public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
 -  
 +
    public static final String ZCONFIG = "/config";
 -  
 +
    public static final String ZTSERVERS = "/tservers";
 -  
 +
    public static final String ZDEAD = "/dead";
 -  public static final String ZDEADTSERVERS = "/dead/tservers";
 -  
 +  public static final String ZDEADTSERVERS = ZDEAD + "/tservers";
 +
    public static final String ZTRACERS = "/tracers";
 -  
 +
    public static final String ZPROBLEMS = "/problems";
 -  public static final String ZUSERS = "/users";
 -  
 +
    public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
 -  
 +
    public static final String ZFATE = "/fate";
 -  
 +
    public static final String ZNEXT_FILE = "/next_file";
 -  
 +
    public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
 -  
 +
    public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
    public static final String ZRECOVERY = "/recovery";
 -  
 -  public static final String METADATA_TABLE_ID = "!0";
 -  public static final String METADATA_TABLE_NAME = "!METADATA";
 -  public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
 -  public static final String TABLE_TABLET_LOCATION = "/table_info";
 -  public static final String ZTABLE_LOCKS = "/table_locks";
 -  
 -  // reserved keyspace is any row that begins with a tilde '~' character
 -  public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
 -  public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
 -  public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
 -  public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
 -  public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
 -  public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
 -  public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
 -  public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
 -  public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
 -  
 -  public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
 -  public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
 -  public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
 -  public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
 -  public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
 -  public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
 -  public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
 -  
 -  // README : very important that prevRow sort last to avoid race conditions between
 -  // garbage collector and split
 -  public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
 -                                                                                                                        // else for that tablet
 -  public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
 -  public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
 -  public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
 -  public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
 -  public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
 -  public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
 -  public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
 -  
 -  public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
 -  public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
 -  public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
 -  public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
 -  public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
 -  
 -  public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
 -      new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
 -  public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
 -  
 -  public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
 -      null);
 -  public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
 -      METADATA_TABLE_ID), null), true);
 -  
 +
    /**
 -   * @deprecated since 1.5.3; not used, but specifies UTF-8
 +   * Initial tablet directory name for the default tablet in all tables
     */
 -  @Deprecated
 -  public static final String VALUE_ENCODING = UTF_8.name();
 -  
 +  public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
 +
 +  public static final String ZTABLE_LOCKS = "/table_locks";
 +
    public static final String BULK_PREFIX = "b-";
 -  public static final String OLD_BULK_PREFIX = "bulk_";
 -  
 -  // note: all times are in milliseconds
 -  
 -  public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
 -  
 -  public static final long MIN_MASTER_LOOP_TIME = 1000;
 -  public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
 -  public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
 -  
 +
 +  public static final String CLONE_PREFIX = "c-";
-   public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF8);
++  public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF_8);
 +
 +  // this affects the table client caching of metadata
 +  public static final int SCAN_BATCH_SIZE = 1000;
 +
 +  // Scanners will default to fetching 3 batches of Key/Value pairs before asynchronously
 +  // fetching the next batch.
 +  public static final long SCANNER_DEFAULT_READAHEAD_THRESHOLD = 3l;
 +
    // Security configuration
    public static final String PW_HASH_ALGORITHM = "SHA-256";
 -  
 -  // Representation of an empty set of authorizations
 -  // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
 -  public static final Authorizations NO_AUTHS = new Authorizations();
 -  
 -  public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
 -  
 +
 +  /**
 +   * @deprecated since 1.6.0; Use {@link Authorizations#EMPTY} instead
 +   */
 +  @Deprecated
 +  public static final Authorizations NO_AUTHS = Authorizations.EMPTY;
 +
    public static final int MAX_DATA_TO_PRINT = 64;
 -  public static final int CLIENT_RETRIES = 5;
 -  public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
    public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
 -  public static final String OLD_PACKAGE_NAME = "cloudbase";
 -  public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
    public static final String MAPFILE_EXTENSION = "map";
    public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
 -  
 +
    public static final String EXPORT_METADATA_FILE = "metadata.bin";
    public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
    public static final String EXPORT_FILE = "exportMetadata.zip";

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
index 35a9067,5134521..041c9e6
--- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
@@@ -16,6 -16,9 +16,8 @@@
   */
  package org.apache.accumulo.core.cli;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import java.util.Iterator;
  import java.util.LinkedHashMap;
  import java.util.Map;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 6d35757,251d6a7..a9c3292
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -16,6 -16,11 +16,8 @@@
   */
  package org.apache.accumulo.core.client;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import java.io.FileNotFoundException;
 -import java.io.IOException;
 -import java.net.UnknownHostException;
  import java.nio.ByteBuffer;
  import java.util.Collections;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
index 63e3497,0000000..6925f18
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
@@@ -1,210 -1,0 +1,212 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.ActiveScan;
 +import org.apache.accumulo.core.client.admin.ActiveCompaction;
 +import org.apache.accumulo.core.client.admin.InstanceOperations;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService;
 +import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +
 +/**
 + * Provides a class for administering the accumulo instance
 + */
 +public class InstanceOperationsImpl implements InstanceOperations {
 +  private Instance instance;
 +  private Credentials credentials;
 +
 +  /**
 +   * @param instance
 +   *          the connection information for this instance
 +   * @param credentials
 +   *          the Credential, containing principal and Authentication Token
 +   */
 +  public InstanceOperationsImpl(Instance instance, Credentials credentials) {
 +    ArgumentChecker.notNull(instance, credentials);
 +    this.instance = instance;
 +    this.credentials = credentials;
 +  }
 +
 +  @Override
 +  public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(property, value);
 +    MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
 +      @Override
 +      public void execute(MasterClientService.Client client) throws Exception {
 +        client.setSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property, value);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(property);
 +    MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
 +      @Override
 +      public void execute(MasterClientService.Client client) throws Exception {
 +        client.removeSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
 +    return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
 +      @Override
 +      public Map<String,String> execute(ClientService.Client client) throws Exception {
 +        return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.CURRENT);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
 +    return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
 +      @Override
 +      public Map<String,String> execute(ClientService.Client client) throws Exception {
 +        return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.SITE);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public List<String> getTabletServers() {
 +    ZooCache cache = new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +    List<String> results = new ArrayList<String>();
 +    for (String candidate : cache.getChildren(path)) {
 +      List<String> children = cache.getChildren(path + "/" + candidate);
 +      if (children != null && children.size() > 0) {
 +        List<String> copy = new ArrayList<String>(children);
 +        Collections.sort(copy);
 +        byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
-         if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
++        if (data != null && !"master".equals(new String(data, UTF_8))) {
 +          results.add(candidate);
 +        }
 +      }
 +    }
 +    return results;
 +  }
 +
 +  @Override
 +  public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
 +    Client client = null;
 +    try {
 +      client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
 +
 +      List<ActiveScan> as = new ArrayList<ActiveScan>();
 +      for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials.toThrift(instance))) {
 +        try {
 +          as.add(new ActiveScanImpl(instance, activeScan));
 +        } catch (TableNotFoundException e) {
 +          throw new AccumuloException(e);
 +        }
 +      }
 +      return as;
 +    } catch (TTransportException e) {
 +      throw new AccumuloException(e);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (TException e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (client != null)
 +        ThriftUtil.returnClient(client);
 +    }
 +  }
 +
 +  @Override
 +  public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
 +    return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
 +      @Override
 +      public Boolean execute(ClientService.Client client) throws Exception {
 +        return client.checkClass(Tracer.traceInfo(), credentials.toThrift(instance), className, asTypeName);
 +      }
 +    });
 +  }
 +
 +  @Override
 +  public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
 +    Client client = null;
 +    try {
 +      client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
 +
 +      List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
 +      for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(),
 +          credentials.toThrift(instance))) {
 +        as.add(new ActiveCompactionImpl(instance, activeCompaction));
 +      }
 +      return as;
 +    } catch (TTransportException e) {
 +      throw new AccumuloException(e);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (TException e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (client != null)
 +        ThriftUtil.returnClient(client);
 +    }
 +  }
 +
 +  @Override
 +  public void ping(String tserver) throws AccumuloException {
 +    TTransport transport = null;
 +    try {
 +      transport = ThriftUtil.createTransport(AddressUtil.parseAddress(tserver, false), ServerConfigurationUtil.getConfiguration(instance));
 +      TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
 +      client.getTabletServerStatus(Tracer.traceInfo(), credentials.toThrift(instance));
 +    } catch (TTransportException e) {
 +      throw new AccumuloException(e);
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloException(e);
 +    } catch (TException e) {
 +      throw new AccumuloException(e);
 +    } finally {
 +      if (transport != null) {
 +        transport.close();
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
index 30f0266,0000000..f32940c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
@@@ -1,154 -1,0 +1,156 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.security.SecurityPermission;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.util.ArgumentChecker.Validator;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +
 +public class Namespaces {
 +  public static final String VALID_NAME_REGEX = "^\\w*$";
 +  public static final Validator<String> VALID_NAME = new Validator<String>() {
 +    @Override
 +    public boolean isValid(String namespace) {
 +      return namespace != null && namespace.matches(VALID_NAME_REGEX);
 +    }
 +
 +    @Override
 +    public String invalidMessage(String namespace) {
 +      if (namespace == null)
 +        return "Namespace cannot be null";
 +      return "Namespaces must only contain word characters (letters, digits, and underscores): " + namespace;
 +    }
 +  };
 +
 +  public static final Validator<String> NOT_DEFAULT = new Validator<String>() {
 +    @Override
 +    public boolean isValid(String namespace) {
 +      return !Namespaces.DEFAULT_NAMESPACE.equals(namespace);
 +    }
 +
 +    @Override
 +    public String invalidMessage(String namespace) {
 +      return "Namespace cannot be the reserved empty namespace";
 +    }
 +  };
 +
 +  public static final Validator<String> NOT_ACCUMULO = new Validator<String>() {
 +    @Override
 +    public boolean isValid(String namespace) {
 +      return !Namespaces.ACCUMULO_NAMESPACE.equals(namespace);
 +    }
 +
 +    @Override
 +    public String invalidMessage(String namespace) {
 +      return "Namespace cannot be the reserved namespace, " + Namespaces.ACCUMULO_NAMESPACE;
 +    }
 +  };
 +
 +  private static SecurityPermission TABLES_PERMISSION = new SecurityPermission("tablesPermission");
 +
 +  public static final String DEFAULT_NAMESPACE_ID = "+default";
 +  public static final String DEFAULT_NAMESPACE = "";
 +  public static final String ACCUMULO_NAMESPACE_ID = "+accumulo";
 +  public static final String ACCUMULO_NAMESPACE = "accumulo";
 +
 +  private static ZooCache getZooCache(Instance instance) {
 +    SecurityManager sm = System.getSecurityManager();
 +    if (sm != null) {
 +      sm.checkPermission(TABLES_PERMISSION);
 +    }
 +    return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +  }
 +
 +  private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
 +    ZooCache zc = getZooCache(instance);
 +
 +    List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
 +
 +    TreeMap<String,String> namespaceMap = new TreeMap<String,String>();
 +
 +    for (String id : namespaceIds) {
 +      byte[] path = zc.get(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + id + Constants.ZNAMESPACE_NAME);
 +      if (path != null) {
 +        if (nameAsKey)
-           namespaceMap.put(new String(path, Constants.UTF8), id);
++          namespaceMap.put(new String(path, UTF_8), id);
 +        else
-           namespaceMap.put(id, new String(path, Constants.UTF8));
++          namespaceMap.put(id, new String(path, UTF_8));
 +      }
 +    }
 +    return namespaceMap;
 +  }
 +
 +  public static boolean exists(Instance instance, String namespaceId) {
 +    ZooCache zc = getZooCache(instance);
 +    List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
 +    return namespaceIds.contains(namespaceId);
 +  }
 +
 +  public static String getNamespaceId(Instance instance, String namespace) throws NamespaceNotFoundException {
 +    String id = getNameToIdMap(instance).get(namespace);
 +    if (id == null)
 +      throw new NamespaceNotFoundException(null, namespace, "getNamespaceId() failed to find namespace");
 +    return id;
 +  }
 +
 +  public static String getNamespaceName(Instance instance, String namespaceId) throws NamespaceNotFoundException {
 +    String namespaceName = getIdToNameMap(instance).get(namespaceId);
 +    if (namespaceName == null)
 +      throw new NamespaceNotFoundException(namespaceId, null, "getNamespaceName() failed to find namespace");
 +    return namespaceName;
 +  }
 +
 +  public static SortedMap<String,String> getNameToIdMap(Instance instance) {
 +    return getMap(instance, true);
 +  }
 +
 +  public static SortedMap<String,String> getIdToNameMap(Instance instance) {
 +    return getMap(instance, false);
 +  }
 +
 +  public static List<String> getTableIds(Instance instance, String namespaceId) throws NamespaceNotFoundException {
 +    String namespace = getNamespaceName(instance, namespaceId);
 +    List<String> names = new LinkedList<String>();
 +    for (Entry<String,String> nameToId : Tables.getNameToIdMap(instance).entrySet())
 +      if (namespace.equals(Tables.qualify(nameToId.getKey()).getFirst()))
 +        names.add(nameToId.getValue());
 +    return names;
 +  }
 +
 +  public static List<String> getTableNames(Instance instance, String namespaceId) throws NamespaceNotFoundException {
 +    String namespace = getNamespaceName(instance, namespaceId);
 +    List<String> names = new LinkedList<String>();
 +    for (String name : Tables.getNameToIdMap(instance).keySet())
 +      if (namespace.equals(Tables.qualify(name).getFirst()))
 +        names.add(name);
 +    return names;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
index 8da530b,1fcebe7..70bdbb0
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
@@@ -16,7 -16,11 +16,9 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.util.ArrayList;
 -import java.util.HashMap;
 -import java.util.Map;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.AccumuloException;
@@@ -128,10 -142,13 +130,10 @@@ public class ServerClient 
      for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
        String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
        byte[] data = ZooUtil.getLockData(zc, path);
-       if (data != null && !new String(data, Constants.UTF8).equals("master"))
 -      if (data != null) {
 -        String tserverData = new String(data, UTF_8);
 -        if (!tserverData.equals("master")) {
 -          servers.add(new ThriftTransportKey(new ServerServices(tserverData).getAddressString(Service.TSERV_CLIENT), conf.getPort(Property.TSERV_CLIENTPORT),
 -              rpcTimeout));
 -        }
 -      }
++      if (data != null && !new String(data, UTF_8).equals("master"))
 +        servers.add(new ThriftTransportKey(
 +          new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT),
 +          rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance))));
      }
      
      boolean opened = false;


[29/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index b1bbb67,0000000..4f381ef
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@@ -1,112 -1,0 +1,114 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +
 +public class RenameTable extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +  private String oldTableName;
 +  private String newTableName;
 +  private String namespaceId;
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.RENAME) + Utils.reserveTable(tableId, tid, true, true, TableOperation.RENAME);
 +  }
 +
 +  public RenameTable(String tableId, String oldTableName, String newTableName) throws NamespaceNotFoundException {
 +    this.tableId = tableId;
 +    this.oldTableName = oldTableName;
 +    this.newTableName = newTableName;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +
 +    Instance instance = master.getInstance();
 +    Pair<String,String> qualifiedOldTableName = Tables.qualify(oldTableName);
 +    Pair<String,String> qualifiedNewTableName = Tables.qualify(newTableName);
 +
 +    // ensure no attempt is made to rename across namespaces
 +    if (newTableName.contains(".") && !namespaceId.equals(Namespaces.getNamespaceId(instance, qualifiedNewTableName.getFirst())))
 +      throw new ThriftTableOperationException(tableId, oldTableName, TableOperation.RENAME, TableOperationExceptionType.INVALID_NAME,
 +          "Namespace in new table name does not match the old table name");
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    Utils.tableNameLock.lock();
 +    try {
 +      Utils.checkTableDoesNotExist(instance, newTableName, tableId, TableOperation.RENAME);
 +
 +      final String newName = qualifiedNewTableName.getSecond();
 +      final String oldName = qualifiedOldTableName.getSecond();
 +
 +      final String tap = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME;
 +
 +      zoo.mutate(tap, null, null, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] current) throws Exception {
-           final String currentName = new String(current, Constants.UTF8);
++          final String currentName = new String(current, UTF_8);
 +          if (currentName.equals(newName))
 +            return null; // assume in this case the operation is running again, so we are done
 +          if (!currentName.equals(oldName)) {
 +            throw new ThriftTableOperationException(null, oldTableName, TableOperation.RENAME, TableOperationExceptionType.NOTFOUND,
 +                "Name changed while processing");
 +          }
-           return newName.getBytes(Constants.UTF8);
++          return newName.getBytes(UTF_8);
 +        }
 +      });
 +      Tables.clearCache(instance);
 +    } finally {
 +      Utils.tableNameLock.unlock();
 +      Utils.unreserveTable(tableId, tid, true);
 +      Utils.unreserveNamespace(this.namespaceId, tid, false);
 +    }
 +
 +    Logger.getLogger(RenameTable.class).debug("Renamed table " + tableId + " " + oldTableName + " " + newTableName);
 +
 +    return null;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveTable(tableId, tid, true);
 +    Utils.unreserveNamespace(namespaceId, tid, false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 7ede305,0000000..3e8704e
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -1,163 -1,0 +1,165 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.math.BigInteger;
 +import java.util.concurrent.locks.Lock;
 +import java.util.concurrent.locks.ReentrantLock;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.fate.zookeeper.ZooReservation;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooQueueLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class Utils {
 +  private static final byte[] ZERO_BYTE = new byte[] {'0'};
 +
 +  static void checkTableDoesNotExist(Instance instance, String tableName, String tableId, TableOperation operation) throws ThriftTableOperationException {
 +
 +    String id = Tables.getNameToIdMap(instance).get(tableName);
 +
 +    if (id != null && !id.equals(tableId))
 +      throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.EXISTS, null);
 +  }
 +
 +  static String getNextTableId(String tableName, Instance instance) throws ThriftTableOperationException {
 +
 +    String tableId = null;
 +    try {
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      final String ntp = ZooUtil.getRoot(instance) + Constants.ZTABLES;
 +      byte[] nid = zoo.mutate(ntp, ZERO_BYTE, ZooUtil.PUBLIC, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] currentValue) throws Exception {
-           BigInteger nextId = new BigInteger(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++          BigInteger nextId = new BigInteger(new String(currentValue, UTF_8), Character.MAX_RADIX);
 +          nextId = nextId.add(BigInteger.ONE);
-           return nextId.toString(Character.MAX_RADIX).getBytes(Constants.UTF8);
++          return nextId.toString(Character.MAX_RADIX).getBytes(UTF_8);
 +        }
 +      });
-       return new String(nid, Constants.UTF8);
++      return new String(nid, UTF_8);
 +    } catch (Exception e1) {
 +      Logger.getLogger(CreateTable.class).error("Failed to assign tableId to " + tableName, e1);
 +      throw new ThriftTableOperationException(tableId, tableName, TableOperation.CREATE, TableOperationExceptionType.OTHER, e1.getMessage());
 +    }
 +  }
 +
 +  static final Lock tableNameLock = new ReentrantLock();
 +  static final Lock idLock = new ReentrantLock();
 +  private static final Logger log = Logger.getLogger(Utils.class);
 +
 +  public static long reserveTable(String tableId, long tid, boolean writeLock, boolean tableMustExist, TableOperation op) throws Exception {
 +    if (getLock(tableId, tid, writeLock).tryLock()) {
 +      if (tableMustExist) {
 +        Instance instance = HdfsZooInstance.getInstance();
 +        IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +        if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId))
 +          throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist");
 +      }
 +      log.info("table " + tableId + " (" + Long.toHexString(tid) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
 +      return 0;
 +    } else
 +      return 100;
 +  }
 +
 +  public static void unreserveTable(String tableId, long tid, boolean writeLock) throws Exception {
 +    getLock(tableId, tid, writeLock).unlock();
 +    log.info("table " + tableId + " (" + Long.toHexString(tid) + ") unlocked for " + (writeLock ? "write" : "read"));
 +  }
 +
 +  public static void unreserveNamespace(String namespaceId, long id, boolean writeLock) throws Exception {
 +    getLock(namespaceId, id, writeLock).unlock();
 +    log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") unlocked for " + (writeLock ? "write" : "read"));
 +  }
 +
 +  public static long reserveNamespace(String namespaceId, long id, boolean writeLock, boolean mustExist, TableOperation op) throws Exception {
 +    if (getLock(namespaceId, id, writeLock).tryLock()) {
 +      if (mustExist) {
 +        Instance instance = HdfsZooInstance.getInstance();
 +        IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +        if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId))
 +          throw new ThriftTableOperationException(namespaceId, "", op, TableOperationExceptionType.NAMESPACE_NOTFOUND, "Namespace does not exist");
 +      }
 +      log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
 +      return 0;
 +    } else
 +      return 100;
 +  }
 +
 +  public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
 +    Instance instance = HdfsZooInstance.getInstance();
 +
 +    String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
-         + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++        + Base64.encodeBase64String(directory.getBytes(UTF_8));
 +
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +
 +    if (ZooReservation.attempt(zk, resvPath, String.format("%016x", tid), "")) {
 +      return 0;
 +    } else
 +      return 50;
 +  }
 +
 +  public static void unreserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
 +    Instance instance = HdfsZooInstance.getInstance();
 +    String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
-         + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++        + Base64.encodeBase64String(directory.getBytes(UTF_8));
 +    ZooReservation.release(ZooReaderWriter.getInstance(), resvPath, String.format("%016x", tid));
 +  }
 +
 +  private static Lock getLock(String tableId, long tid, boolean writeLock) throws Exception {
-     byte[] lockData = String.format("%016x", tid).getBytes(Constants.UTF8);
++    byte[] lockData = String.format("%016x", tid).getBytes(UTF_8);
 +    ZooQueueLock qlock = new ZooQueueLock(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLE_LOCKS + "/" + tableId, false);
 +    Lock lock = DistributedReadWriteLock.recoverLock(qlock, lockData);
 +    if (lock == null) {
 +      DistributedReadWriteLock locker = new DistributedReadWriteLock(qlock, lockData);
 +      if (writeLock)
 +        lock = locker.writeLock();
 +      else
 +        lock = locker.readLock();
 +    }
 +    return lock;
 +  }
 +
 +  public static Lock getReadLock(String tableId, long tid) throws Exception {
 +    return Utils.getLock(tableId, tid, false);
 +  }
 +
 +  static void checkNamespaceDoesNotExist(Instance instance, String namespace, String namespaceId, TableOperation operation)
 +      throws ThriftTableOperationException {
 +
 +    String n = Namespaces.getNameToIdMap(instance).get(namespace);
 +
 +    if (n != null && !n.equals(namespaceId))
 +      throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.NAMESPACE_EXISTS, null);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
index 20b7328,0000000..a9058b7
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
@@@ -1,92 -1,0 +1,94 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tserverOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.master.EventCoordinator.Listener;
 +import org.apache.accumulo.master.tableOps.MasterRepo;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.transport.TTransportException;
 +
 +public class ShutdownTServer extends MasterRepo {
 +  
 +  private static final long serialVersionUID = 1L;
 +  private static final Logger log = Logger.getLogger(ShutdownTServer.class);
 +  private TServerInstance server;
 +  private boolean force;
 +  
 +  public ShutdownTServer(TServerInstance server, boolean force) {
 +    this.server = server;
 +    this.force = force;
 +  }
 +  
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return 0;
 +  }
 +  
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    // suppress assignment of tablets to the server
 +    if (force) {
 +      String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZTSERVERS + "/" + server.getLocation();
 +      ZooLock.deleteLock(path);
 +      path = ZooUtil.getRoot(master.getInstance()) + Constants.ZDEADTSERVERS + "/" + server.getLocation();
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-       zoo.putPersistentData(path, "forced down".getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++      zoo.putPersistentData(path, "forced down".getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +      return null;
 +    }
 +    
 +    // TODO move this to isReady() and drop while loop? - ACCUMULO-1259
 +    Listener listener = master.getEventCoordinator().getListener();
 +    master.shutdownTServer(server);
 +    while (master.onlineTabletServers().contains(server)) {
 +      TServerConnection connection = master.getConnection(server);
 +      if (connection != null) {
 +        try {
 +          TabletServerStatus status = connection.getTableMap(false);
 +          if (status.tableMap != null && status.tableMap.isEmpty()) {
 +            log.info("tablet server hosts no tablets " + server);
 +            connection.halt(master.getMasterLock());
 +            log.info("tablet server asked to halt " + server);
 +            break;
 +          }
 +        } catch (TTransportException ex) {
 +          // expected
 +        } catch (Exception ex) {
 +          log.error("Error talking to tablet server " + server + ": " + ex);
 +        }
 +      }
 +      listener.waitForEvents(1000);
 +    }
 +    
 +    return null;
 +  }
 +  
 +  @Override
 +  public void undo(long tid, Master m) throws Exception {}
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index c4ab6e6,0000000..75d1e9e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -1,817 -1,0 +1,819 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.net.InetAddress;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.MasterClient;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.gc.thrift.GCMonitorService;
 +import org.apache.accumulo.core.gc.thrift.GCStatus;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ServerServices.Service;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.monitor.servlets.DefaultServlet;
 +import org.apache.accumulo.monitor.servlets.GcStatusServlet;
 +import org.apache.accumulo.monitor.servlets.JSONServlet;
 +import org.apache.accumulo.monitor.servlets.LogServlet;
 +import org.apache.accumulo.monitor.servlets.MasterServlet;
 +import org.apache.accumulo.monitor.servlets.OperationServlet;
 +import org.apache.accumulo.monitor.servlets.ProblemServlet;
 +import org.apache.accumulo.monitor.servlets.ScanServlet;
 +import org.apache.accumulo.monitor.servlets.ShellServlet;
 +import org.apache.accumulo.monitor.servlets.TServersServlet;
 +import org.apache.accumulo.monitor.servlets.TablesServlet;
 +import org.apache.accumulo.monitor.servlets.VisServlet;
 +import org.apache.accumulo.monitor.servlets.XMLServlet;
 +import org.apache.accumulo.monitor.servlets.trace.ListType;
 +import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
 +import org.apache.accumulo.monitor.servlets.trace.Summary;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.monitor.LogService;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.problems.ProblemType;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.TableInfoUtil;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * Serve master statistics with an embedded web server.
 + */
 +public class Monitor {
 +  private static final Logger log = Logger.getLogger(Monitor.class);
 +
 +  private static final int REFRESH_TIME = 5;
 +  private static long lastRecalc = 0L;
 +  private static double totalIngestRate = 0.0;
 +  private static double totalIngestByteRate = 0.0;
 +  private static double totalQueryRate = 0.0;
 +  private static double totalScanRate = 0.0;
 +  private static double totalQueryByteRate = 0.0;
 +  private static long totalEntries = 0L;
 +  private static int totalTabletCount = 0;
 +  private static int onlineTabletCount = 0;
 +  private static long totalHoldTime = 0;
 +  private static long totalLookups = 0;
 +  private static int totalTables = 0;
 +
 +  private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
 +    private static final long serialVersionUID = 1L;
 +
 +    private long maxDelta;
 +
 +    public MaxList(long maxDelta) {
 +      this.maxDelta = maxDelta;
 +    }
 +
 +    @Override
 +    public boolean add(Pair<Long,T> obj) {
 +      boolean result = super.add(obj);
 +
 +      if (obj.getFirst() - get(0).getFirst() > maxDelta)
 +        remove(0);
 +
 +      return result;
 +    }
 +
 +  }
 +
 +  private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
 +  private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static EventCounter lookupRateTracker = new EventCounter();
 +  private static EventCounter indexCacheHitTracker = new EventCounter();
 +  private static EventCounter indexCacheRequestTracker = new EventCounter();
 +  private static EventCounter dataCacheHitTracker = new EventCounter();
 +  private static EventCounter dataCacheRequestTracker = new EventCounter();
 +
 +  private static volatile boolean fetching = false;
 +  private static MasterMonitorInfo mmi;
 +  private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
 +  private static Exception problemException;
 +  private static GCStatus gcStatus;
 +
 +  private static Instance instance;
 +
 +  private static ServerConfiguration config;
 +
 +  private static EmbeddedWebServer server;
 +
 +  private ZooLock monitorLock;
 +
 +  private static class EventCounter {
 +
 +    Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
 +    Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
 +    Set<String> serversUpdated = new HashSet<String>();
 +
 +    void startingUpdates() {
 +      serversUpdated.clear();
 +    }
 +
 +    void updateTabletServer(String name, long sampleTime, long numEvents) {
 +      Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
 +      Pair<Long,Long> lastSample = samples.get(name);
 +
 +      if (lastSample == null || !lastSample.equals(newSample)) {
 +        samples.put(name, newSample);
 +        if (lastSample != null) {
 +          prevSamples.put(name, lastSample);
 +        }
 +      }
 +      serversUpdated.add(name);
 +    }
 +
 +    void finishedUpdating() {
 +      // remove any tablet servers not updated
 +      samples.keySet().retainAll(serversUpdated);
 +      prevSamples.keySet().retainAll(serversUpdated);
 +    }
 +
 +    double calculateRate() {
 +      double totalRate = 0;
 +
 +      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
 +        Pair<Long,Long> prevSample = entry.getValue();
 +        Pair<Long,Long> sample = samples.get(entry.getKey());
 +
 +        totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
 +      }
 +
 +      return totalRate;
 +    }
 +
 +    long calculateCount() {
 +      long count = 0;
 +
 +      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
 +        Pair<Long,Long> prevSample = entry.getValue();
 +        Pair<Long,Long> sample = samples.get(entry.getKey());
 +
 +        count += sample.getSecond() - prevSample.getSecond();
 +      }
 +
 +      return count;
 +    }
 +  }
 +
 +  public static void fetchData() {
 +    double totalIngestRate = 0.;
 +    double totalIngestByteRate = 0.;
 +    double totalQueryRate = 0.;
 +    double totalQueryByteRate = 0.;
 +    double totalScanRate = 0.;
 +    long totalEntries = 0;
 +    int totalTabletCount = 0;
 +    int onlineTabletCount = 0;
 +    long totalHoldTime = 0;
 +    long totalLookups = 0;
 +    boolean retry = true;
 +
 +    // only recalc every so often
 +    long currentTime = System.currentTimeMillis();
 +    if (currentTime - lastRecalc < REFRESH_TIME * 1000)
 +      return;
 +
 +    synchronized (Monitor.class) {
 +      if (fetching)
 +        return;
 +      fetching = true;
 +    }
 +
 +    try {
 +      while (retry) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnection(HdfsZooInstance.getInstance());
 +          if (client != null) {
 +            mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
 +            retry = false;
 +          } else {
 +            mmi = null;
 +          }
 +          Monitor.gcStatus = fetchGcStatus();
 +        } catch (Exception e) {
 +          mmi = null;
 +          log.info("Error fetching stats: " + e);
 +        } finally {
 +          if (client != null) {
 +            MasterClient.close(client);
 +          }
 +        }
 +        if (mmi == null)
 +          UtilWaitThread.sleep(1000);
 +      }
 +      if (mmi != null) {
 +        int majorCompactions = 0;
 +        int minorCompactions = 0;
 +
 +        lookupRateTracker.startingUpdates();
 +        indexCacheHitTracker.startingUpdates();
 +        indexCacheRequestTracker.startingUpdates();
 +        dataCacheHitTracker.startingUpdates();
 +        dataCacheRequestTracker.startingUpdates();
 +
 +        for (TabletServerStatus server : mmi.tServerInfo) {
 +          TableInfo summary = TableInfoUtil.summarizeTableStats(server);
 +          totalIngestRate += summary.ingestRate;
 +          totalIngestByteRate += summary.ingestByteRate;
 +          totalQueryRate += summary.queryRate;
 +          totalScanRate += summary.scanRate;
 +          totalQueryByteRate += summary.queryByteRate;
 +          totalEntries += summary.recs;
 +          totalHoldTime += server.holdTime;
 +          totalLookups += server.lookups;
 +          majorCompactions += summary.majors.running;
 +          minorCompactions += summary.minors.running;
 +          lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
 +          indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
 +          indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
 +          dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
 +          dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
 +        }
 +
 +        lookupRateTracker.finishedUpdating();
 +        indexCacheHitTracker.finishedUpdating();
 +        indexCacheRequestTracker.finishedUpdating();
 +        dataCacheHitTracker.finishedUpdating();
 +        dataCacheRequestTracker.finishedUpdating();
 +
 +        int totalTables = 0;
 +        for (TableInfo tInfo : mmi.tableMap.values()) {
 +          totalTabletCount += tInfo.tablets;
 +          onlineTabletCount += tInfo.onlineTablets;
 +          totalTables++;
 +        }
 +        Monitor.totalIngestRate = totalIngestRate;
 +        Monitor.totalTables = totalTables;
 +        totalIngestByteRate = totalIngestByteRate / 1000000.0;
 +        Monitor.totalIngestByteRate = totalIngestByteRate;
 +        Monitor.totalQueryRate = totalQueryRate;
 +        Monitor.totalScanRate = totalScanRate;
 +        totalQueryByteRate = totalQueryByteRate / 1000000.0;
 +        Monitor.totalQueryByteRate = totalQueryByteRate;
 +        Monitor.totalEntries = totalEntries;
 +        Monitor.totalTabletCount = totalTabletCount;
 +        Monitor.onlineTabletCount = onlineTabletCount;
 +        Monitor.totalHoldTime = totalHoldTime;
 +        Monitor.totalLookups = totalLookups;
 +
 +        ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
 +        ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
 +
 +        double totalLoad = 0.;
 +        for (TabletServerStatus status : mmi.tServerInfo) {
 +          if (status != null)
 +            totalLoad += status.osLoad;
 +        }
 +        loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
 +
 +        minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
 +        majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
 +
 +        lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
 +
 +        queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
 +        queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
 +
 +        scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
 +
 +        calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
 +        calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
 +      }
 +      try {
 +        Monitor.problemSummary = ProblemReports.getInstance().summarize();
 +        Monitor.problemException = null;
 +      } catch (Exception e) {
 +        log.info("Failed to obtain problem reports ", e);
 +        Monitor.problemSummary = Collections.emptyMap();
 +        Monitor.problemException = e;
 +      }
 +
 +    } finally {
 +      synchronized (Monitor.class) {
 +        fetching = false;
 +        lastRecalc = currentTime;
 +      }
 +    }
 +  }
 +
 +  private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
 +    long req = cacheReq.calculateCount();
 +    if (req > 0)
 +      hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
 +    else
 +      hitRate.add(new Pair<Long,Double>(currentTime, null));
 +  }
 +
 +  private static GCStatus fetchGcStatus() {
 +    GCStatus result = null;
 +    HostAndPort address = null;
 +    try {
 +      // Read the gc location from its lock
 +      ZooReaderWriter zk = ZooReaderWriter.getInstance();
 +      String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
 +      List<String> locks = zk.getChildren(path, null);
 +      if (locks != null && locks.size() > 0) {
 +        Collections.sort(locks);
-         address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), Constants.UTF8)).getAddress(Service.GC_CLIENT);
++        address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), UTF_8)).getAddress(Service.GC_CLIENT);
 +        GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
 +        try {
 +          result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
 +        } finally {
 +          ThriftUtil.returnClient(client);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      log.warn("Unable to contact the garbage collector at " + address, ex);
 +    }
 +    return result;
 +  }
 +
 +  public static void main(String[] args) throws Exception {
 +    SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +
 +    ServerOpts opts = new ServerOpts();
 +    final String app = "monitor";
 +    opts.parseArgs(app, args);
 +    String hostname = opts.getAddress();
 +
 +    Accumulo.setupLogging(app);
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    instance = HdfsZooInstance.getInstance();
 +    config = new ServerConfiguration(instance);
 +    Accumulo.init(fs, config, app);
 +    Monitor monitor = new Monitor();
 +    Accumulo.enableTracing(hostname, app);
 +    monitor.run(hostname);
 +  }
 +
 +  private static long START_TIME;
 +
 +  public void run(String hostname) {
 +    try {
 +      getMonitorLock();
 +    } catch (Exception e) {
 +      log.error("Failed to get Monitor ZooKeeper lock");
 +      throw new RuntimeException(e);
 +    }
 +
 +    Monitor.START_TIME = System.currentTimeMillis();
 +    int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
 +    try {
 +      log.debug("Creating monitor on port " + port);
 +      server = new EmbeddedWebServer(hostname, port);
 +    } catch (Throwable ex) {
 +      log.error("Unable to start embedded web server", ex);
 +      throw new RuntimeException(ex);
 +    }
 +
 +    server.addServlet(DefaultServlet.class, "/");
 +    server.addServlet(OperationServlet.class, "/op");
 +    server.addServlet(MasterServlet.class, "/master");
 +    server.addServlet(TablesServlet.class, "/tables");
 +    server.addServlet(TServersServlet.class, "/tservers");
 +    server.addServlet(ProblemServlet.class, "/problems");
 +    server.addServlet(GcStatusServlet.class, "/gc");
 +    server.addServlet(LogServlet.class, "/log");
 +    server.addServlet(XMLServlet.class, "/xml");
 +    server.addServlet(JSONServlet.class, "/json");
 +    server.addServlet(VisServlet.class, "/vis");
 +    server.addServlet(ScanServlet.class, "/scans");
 +    server.addServlet(Summary.class, "/trace/summary");
 +    server.addServlet(ListType.class, "/trace/listType");
 +    server.addServlet(ShowTrace.class, "/trace/show");
 +    if (server.isUsingSsl())
 +      server.addServlet(ShellServlet.class, "/shell");
 +    server.start();
 +
 +    try {
 +      hostname = InetAddress.getLocalHost().getHostName();
 +
 +      log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
 +
 +      String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
 +
-       ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(Constants.UTF8),
++      ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(UTF_8),
 +          NodeExistsPolicy.OVERWRITE);
 +      log.info("Set monitor address in zookeeper to " + monitorAddress);
 +    } catch (Exception ex) {
 +      log.error("Unable to set monitor HTTP address in zookeeper", ex);
 +    }
 +
 +    if (null != hostname) {
 +      LogService.startLogListener(Monitor.getSystemConfiguration(), instance.getInstanceID(), hostname);
 +    } else {
 +      log.warn("Not starting log4j listener as we could not determine address to use");
 +    }
 +
 +    new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
 +
 +    // need to regularly fetch data so plot data is updated
 +    new Daemon(new LoggingRunnable(log, new Runnable() {
 +
 +      @Override
 +      public void run() {
 +        while (true) {
 +          try {
 +            Monitor.fetchData();
 +          } catch (Exception e) {
 +            log.warn(e.getMessage(), e);
 +          }
 +
 +          UtilWaitThread.sleep(333);
 +        }
 +
 +      }
 +    }), "Data fetcher").start();
 +
 +    new Daemon(new LoggingRunnable(log, new Runnable() {
 +      @Override
 +      public void run() {
 +        while (true) {
 +          try {
 +            Monitor.fetchScans();
 +          } catch (Exception e) {
 +            log.warn(e.getMessage(), e);
 +          }
 +          UtilWaitThread.sleep(5000);
 +        }
 +      }
 +    }), "Scan scanner").start();
 +  }
 +
 +  public static class ScanStats {
 +    public final List<ActiveScan> scans;
 +    public final long fetched;
 +    ScanStats(List<ActiveScan> active) {
 +      this.scans = active;
 +      this.fetched = System.currentTimeMillis();
 +    }
 +  }
 +  static final Map<String, ScanStats> allScans = new HashMap<String, ScanStats>();
 +  public static Map<String, ScanStats> getScans() {
 +    synchronized (allScans) {
 +      return new TreeMap<String, ScanStats>(allScans);
 +    }
 +  }
 +
 +  protected static void fetchScans() throws Exception {
 +    if (instance == null)
 +      return;
 +    Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
 +    for (String server : c.instanceOperations().getTabletServers()) {
 +      Client tserver = ThriftUtil.getTServerClient(server, Monitor.getSystemConfiguration());
 +      try {
 +        List<ActiveScan> scans = tserver.getActiveScans(null, SystemCredentials.get().toThrift(instance));
 +        synchronized (allScans) {
 +          allScans.put(server, new ScanStats(scans));
 +        }
 +      } catch (Exception ex) {
 +        log.debug(ex, ex);
 +      } finally {
 +        ThriftUtil.returnClient(tserver);
 +      }
 +    }
 +    // Age off old scan information
 +    Iterator<Entry<String,ScanStats>> entryIter = allScans.entrySet().iterator();
 +    long now = System.currentTimeMillis();
 +    while (entryIter.hasNext()) {
 +      Entry<String,ScanStats> entry = entryIter.next();
 +      if (now - entry.getValue().fetched > 5 * 60 * 1000) {
 +        entryIter.remove();
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Get the monitor lock in ZooKeeper
 +   */
 +  private void getMonitorLock() throws KeeperException, InterruptedException {
 +    final String zRoot = ZooUtil.getRoot(instance);
 +    final String monitorPath = zRoot + Constants.ZMONITOR;
 +    final String monitorLockPath = zRoot + Constants.ZMONITOR_LOCK;
 +
 +    // Ensure that everything is kosher with ZK as this has changed.
 +    ZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    if (zoo.exists(monitorPath)) {
 +      byte[] data = zoo.getData(monitorPath, null);
 +      // If the node isn't empty, it's from a previous install (has hostname:port for HTTP server)
 +      if (0 != data.length) {
 +        // Recursively delete from that parent node
 +        zoo.recursiveDelete(monitorPath, NodeMissingPolicy.SKIP);
 +
 +        // And then make the nodes that we expect for the incoming ephemeral nodes
 +        zoo.putPersistentData(monitorPath, new byte[0], NodeExistsPolicy.FAIL);
 +        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
 +      } else if (!zoo.exists(monitorLockPath)) {
 +        // monitor node in ZK exists and is empty as we expect
 +        // but the monitor/lock node does not
 +        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
 +      }
 +    } else {
 +      // 1.5.0 and earlier
 +      zoo.putPersistentData(zRoot + Constants.ZMONITOR, new byte[0], NodeExistsPolicy.FAIL);
 +      if (!zoo.exists(monitorLockPath)) {
 +        // Somehow the monitor node exists but not monitor/lock
 +        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
 +      }
 +    }
 +
 +    // Get a ZooLock for the monitor
 +    while (true) {
 +      MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher();
 +      monitorLock = new ZooLock(monitorLockPath);
 +      monitorLock.lockAsync(monitorLockWatcher, new byte[0]);
 +
 +      monitorLockWatcher.waitForChange();
 +
 +      if (monitorLockWatcher.acquiredLock) {
 +        break;
 +      }
 +
 +      if (!monitorLockWatcher.failedToAcquireLock) {
 +        throw new IllegalStateException("monitor lock in unknown state");
 +      }
 +
 +      monitorLock.tryToCancelAsyncLockOrUnlock();
 +
 +      UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));
 +    }
 +
 +    log.info("Got Monitor lock.");
 +  }
 +
 +  /**
 +   * Async Watcher for monitor lock
 +   */
 +  private static class MoniterLockWatcher implements ZooLock.AsyncLockWatcher {
 +
 +    boolean acquiredLock = false;
 +    boolean failedToAcquireLock = false;
 +
 +    @Override
 +    public void lostLock(LockLossReason reason) {
 +      Halt.halt("Monitor lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
 +    }
 +
 +    @Override
 +    public void unableToMonitorLockNode(final Throwable e) {
 +      Halt.halt(-1, new Runnable() {
 +        @Override
 +        public void run() {
 +          log.fatal("No longer able to monitor Monitor lock node", e);
 +        }
 +      });
 +
 +    }
 +
 +    @Override
 +    public synchronized void acquiredLock() {
 +      if (acquiredLock || failedToAcquireLock) {
 +        Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      acquiredLock = true;
 +      notifyAll();
 +    }
 +
 +    @Override
 +    public synchronized void failedToAcquireLock(Exception e) {
 +      log.warn("Failed to get monitor lock " + e);
 +
 +      if (acquiredLock) {
 +        Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      failedToAcquireLock = true;
 +      notifyAll();
 +    }
 +
 +    public synchronized void waitForChange() {
 +      while (!acquiredLock && !failedToAcquireLock) {
 +        try {
 +          wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +  }
 +
 +  public static MasterMonitorInfo getMmi() {
 +    return mmi;
 +  }
 +
 +  public static int getTotalTables() {
 +    return totalTables;
 +  }
 +
 +  public static int getTotalTabletCount() {
 +    return totalTabletCount;
 +  }
 +
 +  public static int getOnlineTabletCount() {
 +    return onlineTabletCount;
 +  }
 +
 +  public static long getTotalEntries() {
 +    return totalEntries;
 +  }
 +
 +  public static double getTotalIngestRate() {
 +    return totalIngestRate;
 +  }
 +
 +  public static double getTotalIngestByteRate() {
 +    return totalIngestByteRate;
 +  }
 +
 +  public static double getTotalQueryRate() {
 +    return totalQueryRate;
 +  }
 +
 +  public static double getTotalScanRate() {
 +    return totalScanRate;
 +  }
 +
 +  public static double getTotalQueryByteRate() {
 +    return totalQueryByteRate;
 +  }
 +
 +  public static long getTotalHoldTime() {
 +    return totalHoldTime;
 +  }
 +
 +  public static Exception getProblemException() {
 +    return problemException;
 +  }
 +
 +  public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
 +    return problemSummary;
 +  }
 +
 +  public static GCStatus getGcStatus() {
 +    return gcStatus;
 +  }
 +
 +  public static long getTotalLookups() {
 +    return totalLookups;
 +  }
 +
 +  public static long getStartTime() {
 +    return START_TIME;
 +  }
 +
 +  public static List<Pair<Long,Double>> getLoadOverTime() {
 +    synchronized (loadOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(loadOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getIngestRateOverTime() {
 +    synchronized (ingestRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
 +    synchronized (ingestByteRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
 +    synchronized (recoveriesOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
 +    synchronized (minorCompactionsOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
 +    synchronized (majorCompactionsOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getLookupsOverTime() {
 +    synchronized (lookupsOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
 +    }
 +  }
 +
 +  public static double getLookupRate() {
 +    return lookupRateTracker.calculateRate();
 +  }
 +
 +  public static List<Pair<Long,Integer>> getQueryRateOverTime() {
 +    synchronized (queryRateOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getScanRateOverTime() {
 +    synchronized (scanRateOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
 +    synchronized (queryByteRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
 +    synchronized (indexCacheHitRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
 +    synchronized (dataCacheHitRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
 +    }
 +  }
 +
 +  public static AccumuloConfiguration getSystemConfiguration() {
 +    return config.getConfiguration();
 +  }
 +
 +  public static Instance getInstance() {
 +    return instance;
 +  }
 +
 +  public static boolean isUsingSsl() {
 +    return server.isUsingSsl();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index 5e3e475,0000000..78dd95e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -1,140 -1,0 +1,141 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collection;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.util.TTimeoutTransport;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class ZooKeeperStatus implements Runnable {
 +  
 +  private static final Logger log = Logger.getLogger(ZooKeeperStatus.class);
 +  
 +  private volatile boolean stop = false;
 +  
 +  public static class ZooKeeperState implements Comparable<ZooKeeperState> {
 +    public final String keeper;
 +    public final String mode;
 +    public final int clients;
 +    
 +    public ZooKeeperState(String keeper, String mode, int clients) {
 +      this.keeper = keeper;
 +      this.mode = mode;
 +      this.clients = clients;
 +    }
 +    
 +    @Override
 +    public int compareTo(ZooKeeperState other) {
 +      if (this == other) {
 +        return 0;
 +      } else if (other == null) {
 +        return 1;
 +      } else {
 +        if (this.keeper == other.keeper) {
 +          return 0;
 +        } else if (null == this.keeper) {
 +          return -1;
 +        } else if (null == other.keeper) {
 +          return 1;
 +        } else {
 +          return this.keeper.compareTo(other.keeper);
 +        }
 +      }
 +    }
 +  }
 +  
 +  private static SortedSet<ZooKeeperState> status = new TreeSet<ZooKeeperState>();
 +  
 +  public static Collection<ZooKeeperState> getZooKeeperStatus() {
 +    return status;
 +  }
 +  
 +  public void stop() {
 +    this.stop = true;
 +  }
 +  
 +  @Override
 +  public void run() {
 +    
 +    while (!stop) {
 +      
 +      TreeSet<ZooKeeperState> update = new TreeSet<ZooKeeperState>();
 +      
 +      String zookeepers[] = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST).split(",");
 +      for (String keeper : zookeepers) {
 +        int clients = 0;
 +        String mode = "unknown";
 +        
 +        String[] parts = keeper.split(":");
 +        TTransport transport = null;
 +        try {
 +          HostAndPort addr;
 +          if (parts.length > 1)
 +            addr = HostAndPort.fromParts(parts[0], Integer.parseInt(parts[1]));
 +          else
 +            addr = HostAndPort.fromParts(parts[0], 2181);
 +          
 +          transport = TTimeoutTransport.create(addr, 10 * 1000l);
-           transport.write("stat\n".getBytes(Constants.UTF8), 0, 5);
++          transport.write("stat\n".getBytes(UTF_8), 0, 5);
 +          StringBuilder response = new StringBuilder();
 +          try {
 +            transport.flush();
 +            byte[] buffer = new byte[1024 * 100];
 +            int n = 0;
 +            while ((n = transport.read(buffer, 0, buffer.length)) > 0) {
-               response.append(new String(buffer, 0, n, Constants.UTF8));
++              response.append(new String(buffer, 0, n, UTF_8));
 +            }
 +          } catch (TTransportException ex) {
 +            // happens at EOF
 +          }
 +          for (String line : response.toString().split("\n")) {
 +            if (line.startsWith(" "))
 +              clients++;
 +            if (line.startsWith("Mode"))
 +              mode = line.split(":")[1];
 +          }
 +          update.add(new ZooKeeperState(keeper, mode, clients));
 +        } catch (Exception ex) {
 +          log.info("Exception talking to zookeeper " + keeper, ex);
 +          update.add(new ZooKeeperState(keeper, "Down", -1));
 +        } finally {
 +          if (transport != null) {
 +            try {
 +              transport.close();
 +            } catch (Exception ex) {
 +              log.error(ex, ex);
 +            }
 +          }
 +        }
 +      }
 +      status = update;
 +      UtilWaitThread.sleep(5 * 1000);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index d896040,0000000..7812603
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@@ -1,283 -1,0 +1,285 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.PrintWriter;
 +import java.io.StringWriter;
 +import java.io.UnsupportedEncodingException;
 +import java.net.URLDecoder;
 +import java.net.URLEncoder;
 +import java.util.Date;
 +import java.util.List;
 +import java.util.TimerTask;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.http.Cookie;
 +import javax.servlet.http.HttpServlet;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.monitor.DedupedLogEvent;
 +import org.apache.accumulo.server.monitor.LogService;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +abstract public class BasicServlet extends HttpServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +  protected static final Logger log = Logger.getLogger(BasicServlet.class);
 +  static String cachedInstanceName = null;
 +  private static String bannerText;
 +  private static String bannerColor;
 +  private static String bannerBackground;
 +
 +  abstract protected String getTitle(HttpServletRequest req);
 +
 +  @Override
 +  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    StringBuilder sb = new StringBuilder();
 +    try {
 +      Monitor.fetchData();
 +      bannerText = sanitize(Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_TEXT));
 +      bannerColor = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_COLOR).replace("'", "&#39;");
 +      bannerBackground = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_BACKGROUND).replace("'", "&#39;");
 +      pageStart(req, resp, sb);
 +      pageBody(req, resp, sb);
 +      pageEnd(req, resp, sb);
 +    } catch (Throwable t) {
 +      log.error("Error building page " + req.getRequestURI(), t);
 +      sb.append("\n<pre>\n");
 +      StringWriter sw = new StringWriter();
 +      t.printStackTrace(new PrintWriter(sw));
 +      sb.append(sanitize(sw.getBuffer().toString()));
 +      sb.append("</pre>\n");
 +    } finally {
 +      resp.getWriter().print(sb);
 +      resp.getWriter().flush();
 +    }
 +  }
 +
 +  @Override
 +  protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    doGet(req, resp);
 +  }
 +
 +  private static final String DEFAULT_CONTENT_TYPE = "text/html";
 +
 +  public static final void setCookie(HttpServletResponse resp, String name, String value) {
 +    resp.addCookie(new Cookie(name, value));
 +  }
 +
 +  public static final String getCookieValue(HttpServletRequest req, String name) {
 +    if (req.getCookies() != null)
 +      for (Cookie c : req.getCookies())
 +        if (c.getName().equals(name))
 +          return c.getValue();
 +    return null;
 +  }
 +
 +  protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
 +    resp.setContentType(DEFAULT_CONTENT_TYPE);
 +    int refresh = -1;
 +    String refreshStr = getCookieValue(req, "page.refresh.rate");
 +    if (refreshStr != null) {
 +      try {
 +        refresh = Integer.parseInt(BasicServlet.decode(refreshStr));
 +      } catch (NumberFormatException e) {
 +        // ignore improperly formatted user cookie
 +      }
 +    }
 +    synchronized (BasicServlet.class) {
 +      // Learn our instance name asynchronously so we don't hang up if zookeeper is down
 +      if (cachedInstanceName == null) {
 +        SimpleTimer.getInstance().schedule(new TimerTask() {
 +          @Override
 +          public void run() {
 +            synchronized (BasicServlet.class) {
 +              if (cachedInstanceName == null) {
 +                cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName();
 +              }
 +            }
 +          }
 +        }, 1000);
 +      }
 +    }
 +
 +    // BEGIN PAGE
 +    sb.append("<!--\n");
 +    sb.append("  Licensed to the Apache Software Foundation (ASF) under one or more\n");
 +    sb.append("  contributor license agreements.  See the NOTICE file distributed with\n");
 +    sb.append("  this work for additional information regarding copyright ownership.\n");
 +    sb.append("  The ASF licenses this file to You under the Apache License, Version 2.0\n");
 +    sb.append("  (the \"License\"); you may not use this file except in compliance with\n");
 +    sb.append("  the License.  You may obtain a copy of the License at\n");
 +    sb.append("\n");
 +    sb.append("    http://www.apache.org/licenses/LICENSE-2.0\n");
 +    sb.append("\n");
 +    sb.append("  Unless required by applicable law or agreed to in writing, software\n");
 +    sb.append("  distributed under the License is distributed on an \"AS IS\" BASIS,\n");
 +    sb.append("  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n");
 +    sb.append("  See the License for the specific language governing permissions and\n");
 +    sb.append("  limitations under the License.\n");
 +    sb.append("-->\n");
 +    sb.append("<html>\n");
 +
 +    // BEGIN HEADER
 +    sb.append("<head>\n");
 +    sb.append("<title>").append(getTitle(req)).append(" - Accumulo ").append(Constants.VERSION).append("</title>\n");
 +    if ((refresh > 0) && (req.getRequestURI().startsWith("/vis") == false) && (req.getRequestURI().startsWith("/shell") == false))
 +      sb.append("<meta http-equiv='refresh' content='" + refresh + "' />\n");
 +    sb.append("<meta http-equiv='Content-Type' content='").append(DEFAULT_CONTENT_TYPE).append("' />\n");
 +    sb.append("<meta http-equiv='Content-Script-Type' content='text/javascript' />\n");
 +    sb.append("<meta http-equiv='Content-Style-Type' content='text/css' />\n");
 +    sb.append("<link rel='shortcut icon' type='image/jpg' href='/web/favicon.png' />\n");
 +    sb.append("<link rel='stylesheet' type='text/css' href='/web/screen.css' media='screen' />\n");
 +    sb.append("<script src='/web/functions.js' type='text/javascript'></script>\n");
 +
 +    sb.append("<!--[if lte IE 8]><script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/excanvas.min.js\"></script><![endif]-->\n");
 +    sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.js\"></script>\n");
 +    sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.flot.js\"></script>\n");
 +
 +    sb.append("</head>\n");
 +
 +    // BEGIN BODY OPENING
 +    sb.append("\n<body ").append(getBodyAttributes()).append(">\n");
 +    sb.append("<div id='content-wrapper'>\n");
 +    sb.append("<div id='content'>\n");
 +    sb.append("<div id='header'>");
 +    if (!bannerText.isEmpty()) {
 +      sb.append("<div id='banner' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
 +          .append("</div>\n");
 +    }
 +    sb.append("<div id='headertitle'>");
 +    sb.append("<h1>").append(getTitle(req)).append("</h1></div>\n");
 +    sb.append("<div id='subheader'>Instance&nbsp;Name:&nbsp;").append(cachedInstanceName).append("&nbsp;&nbsp;&nbsp;Version:&nbsp;").append(Constants.VERSION)
 +        .append("\n");
 +    sb.append("<br><span class='smalltext'>Instance&nbsp;ID:&nbsp;").append(HdfsZooInstance.getInstance().getInstanceID()).append("</span>\n");
 +    sb.append("<br><span class='smalltext'>").append(new Date().toString().replace(" ", "&nbsp;")).append("</span>");
 +    sb.append("</div>\n"); // end <div id='subheader'>
 +    sb.append("</div>\n"); // end <div id='header'>
 +
 +    // BEGIN LEFT SIDE
 +    sb.append("<div id='nav'>\n");
 +    sb.append("<span id='nav-title'><a href='/'>Overview</a></span><br />\n");
 +    sb.append("<hr />\n");
 +    sb.append("<a href='/master'>Master&nbsp;Server</a><br />\n");
 +    sb.append("<a href='/tservers'>Tablet&nbsp;Servers</a><br />\n");
 +    sb.append("<a href='/scans'>Active&nbsp;Scans</a><br />\n");
 +    sb.append("<a href='/vis'>Server Activity</a><br />\n");
 +    sb.append("<a href='/gc'>Garbage&nbsp;Collector</a><br />\n");
 +    sb.append("<a href='/tables'>Tables</a><br />\n");
 +    sb.append("<a href='/trace/summary?minutes=10'>Recent&nbsp;Traces</a><br />\n");
 +    List<DedupedLogEvent> dedupedLogEvents = LogService.getInstance().getEvents();
 +    int numLogs = dedupedLogEvents.size();
 +    boolean logsHaveError = false;
 +    for (DedupedLogEvent dedupedLogEvent : dedupedLogEvents)
 +      if (dedupedLogEvent.getEvent().getLevel().isGreaterOrEqual(Level.ERROR)) {
 +        logsHaveError = true;
 +        break;
 +      }
 +    if (numLogs > 0)
 +      sb.append("<span class='" + (logsHaveError ? "error" : "warning") + "'><a href='/log'>Recent&nbsp;Logs&nbsp;<span class='smalltext'>(" + numLogs
 +          + ")</a></span></span><br />\n");
 +    int numProblems = Monitor.getProblemSummary().entrySet().size();
 +    if (numProblems > 0)
 +      sb.append("<span class='error'><a href='/problems'>Table&nbsp;Problems&nbsp;<span class='smalltext'>(" + numProblems + ")</a></span></span><br />\n");
 +    sb.append("<hr />\n");
 +    sb.append("<a href='/xml'>XML</a><br />\n");
 +    sb.append("<a href='/json'>JSON</a><hr />\n");
 +    if (Monitor.isUsingSsl())
 +      sb.append("<a href='/shell'>Shell</a><hr />\n");
 +    sb.append("<div class='smalltext'>[<a href='").append("/op?action=refresh&value=").append(refresh < 1 ? "5" : "-1");
 +    sb.append("&redir=").append(currentPage(req)).append("'>");
 +    sb.append(refresh < 1 ? "en" : "dis").append("able&nbsp;auto-refresh</a>]</div>\n");
 +    sb.append("</div>\n"); // end <div id='nav'>
 +
 +    sb.append("<div id='main'");
 +    if (bannerText.isEmpty())
 +      sb.append(" style='bottom:0'");
 +    sb.append(">\n");
 +    sb.append("<!-- BEGIN MAIN BODY CONTENT -->\n\n");
 +  }
 +
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
 +    sb.append("This page intentionally left blank.");
 +  }
 +
 +  protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
 +    sb.append("\n<!-- END MAIN BODY CONTENT -->\n");
 +    sb.append("</div>\n"); // end <div id='main'>
 +
 +    // BEGIN FOOTER
 +    sb.append("</div>\n"); // end <div id='content'>
 +    sb.append("</div>\n"); // end <div id='content-wrapper'>
 +    if (!bannerText.isEmpty()) {
 +      sb.append("<div id='footer' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
 +          .append("</div>\n");
 +    }
 +    sb.append("</body>\n");
 +    sb.append("</html>\n");
 +  }
 +
 +  /**
 +   * Allow the concrete servlet implementation to provide attributes on the body HTML tag, such as 'onload', which can be used to call Javascript methods on
 +   * page load. By default, nothing is specified.
 +   */
 +  protected String getBodyAttributes() {
 +    return "";
 +  }
 +
 +  public static String encode(String s) {
 +    try {
-       return URLEncoder.encode(s, Constants.UTF8.name());
++      return URLEncoder.encode(s, UTF_8.name());
 +    } catch (UnsupportedEncodingException e) {
-       Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++      Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
 +      throw new AssertionError(e); // can't happen with UTF-8
 +    }
 +  }
 +
 +  public static String decode(String s) {
 +    try {
-       return URLDecoder.decode(s, Constants.UTF8.name());
++      return URLDecoder.decode(s, UTF_8.name());
 +    } catch (UnsupportedEncodingException e) {
-       Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++      Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
 +      throw new AssertionError(e); // can't happen with UTF-8
 +    }
 +  }
 +
 +  public static String sanitize(String xml) {
 +    return xml.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
 +  }
 +
 +  public static String currentPage(HttpServletRequest req) {
 +    String redir = req.getRequestURI();
 +    if (req.getQueryString() != null)
 +      redir += "?" + req.getQueryString();
 +    return encode(redir);
 +  }
 +
 +  protected static void banner(StringBuilder sb, String klass, String text) {
 +    sb.append("<br />\n<h2 class='").append(klass).append("'>").append(text).append("</h2>\n");
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index d88bd7c,0000000..bb7e690
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -1,346 -1,0 +1,347 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Calendar;
 +import java.util.Date;
 +import java.util.List;
 +import java.util.TimeZone;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.ServletOutputStream;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.util.Duration;
 +import org.apache.accumulo.core.util.NumUtil;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.ZooKeeperStatus;
 +import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
 +import org.apache.accumulo.monitor.util.celltypes.NumberType;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.hadoop.fs.ContentSummary;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +
 +public class DefaultServlet extends BasicServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Accumulo Overview";
 +  }
 +
 +  private void getResource(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 +    try {
 +      String path = req.getRequestURI();
 +
 +      if (path.endsWith(".jpg"))
 +        resp.setContentType("image/jpeg");
 +
 +      if (path.endsWith(".html"))
 +        resp.setContentType("text/html");
 +
 +      path = path.substring(1);
 +      InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path);
 +      ServletOutputStream out = resp.getOutputStream();
 +      try {
 +        if (data != null) {
 +          byte[] buffer = new byte[1024];
 +          int n;
 +          while ((n = data.read(buffer)) > 0)
 +            out.write(buffer, 0, n);
 +        } else {
-           out.write(("could not get resource " + path + "").getBytes(Constants.UTF8));
++          out.write(("could not get resource " + path + "").getBytes(UTF_8));
 +        }
 +      } finally {
 +        if (data != null)
 +          data.close();
 +      }
 +    } catch (Throwable t) {
 +      log.error(t, t);
 +      throw new IOException(t);
 +    }
 +  }
 +
 +  @Override
 +  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    if (req.getRequestURI().startsWith("/web"))
 +      getResource(req, resp);
 +    else if (req.getRequestURI().startsWith("/monitor"))
 +      resp.sendRedirect("/master");
 +    else if (req.getRequestURI().startsWith("/errors"))
 +      resp.sendRedirect("/problems");
 +    else
 +      super.doGet(req, resp);
 +  }
 +
 +  public static final int GRAPH_WIDTH = 450;
 +  public static final int GRAPH_HEIGHT = 150;
 +
 +  private static void plotData(StringBuilder sb, String title, @SuppressWarnings("rawtypes") List data, boolean points) {
 +    plotData(sb, title, points, new ArrayList<String>(), data);
 +  }
 +
 +  @SuppressWarnings("rawtypes")
 +  private static void plotData(StringBuilder sb, String title, boolean points, List<String> labels, List... series) {
 +    sb.append("<div class=\"plotHeading\">");
 +    sb.append(title);
 +    sb.append("</div>");
 +    sb.append("</br>");
 +    String id = "c" + title.hashCode();
 +    sb.append("<div id=\"" + id + "\" style=\"width:" + GRAPH_WIDTH + "px;height:" + GRAPH_HEIGHT + "px;\"></div>\n");
 +
 +    sb.append("<script type=\"text/javascript\">\n");
 +    sb.append("$(function () {\n");
 +
 +    for (int i = 0; i < series.length; i++) {
 +
 +      @SuppressWarnings("unchecked")
 +      List<Pair<Long,? extends Number>> data = series[i];
 +      sb.append("    var d" + i + " = [");
 +
 +      String sep = "";
 +      for (Pair<Long,? extends Number> point : data) {
 +        if (point.getSecond() == null)
 +          continue;
 +
 +        String y;
 +        if (point.getSecond() instanceof Double)
 +          y = String.format("%1.2f", point.getSecond());
 +        else
 +          y = point.getSecond().toString();
 +
 +        sb.append(sep);
 +        sep = ",";
 +        sb.append("[" + utc2local(point.getFirst()) + "," + y + "]");
 +      }
 +      sb.append("    ];\n");
 +    }
 +
 +    String opts = "lines: { show: true }";
 +    if (points)
 +      opts = "points: { show: true, radius: 1 }";
 +
 +    sb.append("    $.plot($(\"#" + id + "\"),");
 +    String sep = "";
 +
 +    String colors[] = new String[] {"red", "blue", "green", "black"};
 +
 +    sb.append("[");
 +    for (int i = 0; i < series.length; i++) {
 +      sb.append(sep);
 +      sep = ",";
 +      sb.append("{ ");
 +      if (labels.size() > 0) {
 +        sb.append("label: \"" + labels.get(i) + "\", ");
 +      }
 +      sb.append("data: d" + i + ", " + opts + ", color:\"" + colors[i] + "\" }");
 +    }
 +    sb.append("], ");
 +    sb.append("{yaxis:{}, xaxis:{mode:\"time\",minTickSize: [1, \"minute\"],timeformat: \"%H:%M<br />" + getShortTZName() + "\", ticks:3}});");
 +    sb.append("   });\n");
 +    sb.append("</script>\n");
 +  }
 +
 +  /**
 +   * Shows the current time zone (based on the current time) short name
 +   */
 +  private static String getShortTZName() {
 +    TimeZone tz = TimeZone.getDefault();
 +    return tz.getDisplayName(tz.inDaylightTime(new Date()), TimeZone.SHORT);
 +  }
 +
 +  /**
 +   * Converts a unix timestamp in UTC to one that is relative to the local timezone
 +   */
 +  private static Long utc2local(Long utcMillis) {
 +    Calendar currentCalendar = Calendar.getInstance(); // default timezone
 +    currentCalendar.setTimeInMillis(utcMillis + currentCalendar.getTimeZone().getOffset(utcMillis));
 +    return currentCalendar.getTime().getTime();
 +  }
 +
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws IOException {
 +
 +    sb.append("<table class='noborder'>\n");
 +    sb.append("<tr>\n");
 +
 +    sb.append("<td class='noborder'>\n");
 +    doAccumuloTable(sb);
 +    sb.append("</td>\n");
 +
 +    sb.append("<td class='noborder'>\n");
 +    doZooKeeperTable(sb);
 +    sb.append("</td>\n");
 +
 +    sb.append("</tr></table>\n");
 +    sb.append("<br/>\n");
 +
 +    sb.append("<p/><table class=\"noborder\">\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Ingest (Entries/s)", Monitor.getIngestRateOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Scan (Entries/s)", false, Arrays.asList("Read", "Returned"), Monitor.getScanRateOverTime(), Monitor.getQueryRateOverTime());
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Ingest (MB/s)", Monitor.getIngestByteRateOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Scan (MB/s)", Monitor.getQueryByteRateOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Load Average", Monitor.getLoadOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Seeks", Monitor.getLookupsOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Minor Compactions", Monitor.getMinorCompactionsOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Major Compactions", Monitor.getMajorCompactionsOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Index Cache Hit Rate", Monitor.getIndexCacheHitRateOverTime(), true);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Data Cache Hit Rate", Monitor.getDataCacheHitRateOverTime(), true);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("</table>\n");
 +  }
 +
 +  private void doAccumuloTable(StringBuilder sb) throws IOException {
 +    // Accumulo
 +    VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
 +    MasterMonitorInfo info = Monitor.getMmi();
 +    sb.append("<table>\n");
 +    sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
 +    if (info == null) {
 +      sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
 +    } else {
 +      long totalAcuBytesUsed = 0l;
 +      long totalHdfsBytesUsed = 0l;
 +
 +      try {
 +        for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
 +          final Path basePath = new Path(baseDir);
 +          final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
 +
 +          try {
 +            // Calculate the amount of space used by Accumulo on the FileSystem
 +            ContentSummary accumuloSummary = fs.getContentSummary(basePath);
 +            long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed();
 +            totalAcuBytesUsed += bytesUsedByAcuOnFs;
 +
 +            // Catch the overflow -- this is big data
 +            if (totalAcuBytesUsed < bytesUsedByAcuOnFs) {
 +              log.debug("Overflowed long in bytes used by Accumulo for " + baseDir);
 +              totalAcuBytesUsed = 0l;
 +              break;
 +            }
 +
 +            // Calculate the total amount of space used on the FileSystem
 +            ContentSummary volumeSummary = fs.getContentSummary(new Path("/"));
 +            long bytesUsedOnVolume = volumeSummary.getSpaceConsumed();
 +            totalHdfsBytesUsed += bytesUsedOnVolume;
 +
 +            // Catch the overflow -- this is big data
 +            if (totalHdfsBytesUsed < bytesUsedOnVolume) {
 +              log.debug("Overflowed long in bytes used in HDFS for " + baseDir);
 +              totalHdfsBytesUsed = 0;
 +              break;
 +            }
 +          } catch (Exception ex) {
 +            log.trace("Unable to get disk usage information for " + baseDir, ex);
 +          }
 +        }
 +
 +        String diskUsed = "Unknown";
 +        String consumed = null;
 +        if (totalAcuBytesUsed > 0) {
 +          // Convert Accumulo usage to a readable String
 +          diskUsed = bytes(totalAcuBytesUsed);
 +
 +          if (totalHdfsBytesUsed > 0) {
 +            // Compute amount of space used by Accumulo as a percentage of total space usage.
 +            consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed);
 +          }
 +        }
 +
 +        boolean highlight = false;
 +        tableRow(sb, (highlight = !highlight), "Disk&nbsp;Used", diskUsed);
 +        if (null != consumed)
 +          tableRow(sb, (highlight = !highlight), "%&nbsp;of&nbsp;Used&nbsp;DFS", consumed);
 +        tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables()));
 +        tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet&nbsp;Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));
 +        tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Dead&nbsp;Tablet&nbsp;Servers</a>", NumberType.commas(info.deadTabletServers.size(), 0, 0));
 +        tableRow(sb, (highlight = !highlight), "Tablets", NumberType.commas(Monitor.getTotalTabletCount(), 1, Long.MAX_VALUE));
 +        tableRow(sb, (highlight = !highlight), "Entries", NumberType.commas(Monitor.getTotalEntries()));
 +        tableRow(sb, (highlight = !highlight), "Lookups", NumberType.commas(Monitor.getTotalLookups()));
 +        tableRow(sb, (highlight = !highlight), "Uptime", Duration.format(System.currentTimeMillis() - Monitor.getStartTime()));
 +      } catch (Exception e) {
 +        log.debug(e, e);
 +      }
 +    }
 +    sb.append("</table>\n");
 +  }
 +
 +  private void doZooKeeperTable(StringBuilder sb) throws IOException {
 +    // Zookeepers
 +    sb.append("<table>\n");
 +    sb.append("<tr><th colspan='3'>Zookeeper</th></tr>\n");
 +    sb.append("<tr><th>Server</th><th>Mode</th><th>Clients</th></tr>\n");
 +
 +    boolean highlight = false;
 +    for (ZooKeeperState k : ZooKeeperStatus.getZooKeeperStatus()) {
 +      if (k.clients >= 0) {
 +        tableRow(sb, (highlight = !highlight), k.keeper, k.mode, k.clients);
 +      } else {
 +        tableRow(sb, false, k.keeper, "<span class='error'>Down</span>", "");
 +      }
 +    }
 +    sb.append("</table>\n");
 +  }
 +
 +  private static String bytes(long big) {
 +    return NumUtil.bigNumberForSize(big);
 +  }
 +
 +  public static void tableRow(StringBuilder sb, boolean highlight, Object... cells) {
 +    sb.append(highlight ? "<tr class='highlight'>" : "<tr>");
 +    for (int i = 0; i < cells.length; ++i) {
 +      Object cell = cells[i];
 +      String cellValue = cell == null ? "" : String.valueOf(cell).trim();
 +      sb.append("<td class='").append(i < cells.length - 1 ? "left" : "right").append("'>").append(cellValue.isEmpty() ? "-" : cellValue).append("</td>");
 +    }
 +    sb.append("</tr>\n");
 +  }
 +}


[16/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 1e7658a,0000000..6c40094
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -1,329 -1,0 +1,331 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.File;
 +import java.io.FileInputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.net.InetAddress;
 +import java.net.UnknownHostException;
 +import java.util.Map.Entry;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.trace.DistributedTrace;
 +import org.apache.accumulo.core.util.AddressUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.util.Version;
 +import org.apache.accumulo.core.volume.Volume;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.ReadOnlyStore;
 +import org.apache.accumulo.fate.ReadOnlyTStore;
 +import org.apache.accumulo.fate.ZooStore;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.watcher.Log4jConfiguration;
 +import org.apache.accumulo.server.watcher.MonitorLog4jWatcher;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.helpers.LogLog;
 +import org.apache.log4j.xml.DOMConfigurator;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class Accumulo {
 +
 +  private static final Logger log = Logger.getLogger(Accumulo.class);
 +
 +  public static synchronized void updateAccumuloVersion(VolumeManager fs, int oldVersion) {
 +    for (Volume volume : fs.getVolumes()) {
 +      try {
 +        if (getAccumuloPersistentVersion(fs) == oldVersion) {
 +          log.debug("Attempting to upgrade " + volume);
 +          Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
 +          fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
 +          // TODO document failure mode & recovery if FS permissions cause above to work and below to fail ACCUMULO-2596
 +          Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(oldVersion));
 +          if (!fs.delete(prevDataVersionLoc)) {
 +            throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
 +          }
 +        }
 +      } catch (IOException e) {
 +        throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
 +      }
 +    }
 +  }
 +
 +  public static synchronized int getAccumuloPersistentVersion(FileSystem fs, Path path) {
 +    int dataVersion;
 +    try {
 +      FileStatus[] files = fs.listStatus(path);
 +      if (files == null || files.length == 0) {
 +        dataVersion = -1; // assume it is 0.5 or earlier
 +      } else {
 +        dataVersion = Integer.parseInt(files[0].getPath().getName());
 +      }
 +      return dataVersion;
 +    } catch (IOException e) {
 +      throw new RuntimeException("Unable to read accumulo version: an error occurred.", e);
 +    }
 +  }
 +
 +  public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
 +    // It doesn't matter which Volume is used as they should all have the data version stored
 +    Volume v = fs.getVolumes().iterator().next();
 +    Path path = ServerConstants.getDataVersionLocation(v);
 +    return getAccumuloPersistentVersion(v.getFileSystem(), path);
 +  }
 +
 +  public static synchronized Path getAccumuloInstanceIdPath(VolumeManager fs) {
 +    // It doesn't matter which Volume is used as they should all have the instance ID stored
 +    Volume v = fs.getVolumes().iterator().next();
 +    return ServerConstants.getInstanceIdLocation(v);
 +  }
 +
 +  public static void enableTracing(String address, String application) {
 +    try {
 +      DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address);
 +    } catch (Exception ex) {
 +      log.error("creating remote sink for trace spans", ex);
 +    }
 +  }
 +
 +  /**
 +   * Finds the best log4j configuration file. A generic file is used only if an
 +   * application-specific file is not available. An XML file is preferred over
 +   * a properties file, if possible.
 +   *
 +   * @param confDir directory where configuration files should reside
 +   * @param application application name for configuration file name
 +   * @return configuration file name
 +   */
 +  static String locateLogConfig(String confDir, String application) {
 +    String explicitConfigFile = System.getProperty("log4j.configuration");
 +    if (explicitConfigFile != null) {
 +      return explicitConfigFile;
 +    }
 +    String[] configFiles = {
 +      String.format("%s/%s_logger.xml", confDir, application),
 +      String.format("%s/%s_logger.properties", confDir, application),
 +      String.format("%s/generic_logger.xml", confDir),
 +      String.format("%s/generic_logger.properties", confDir)
 +    };
 +    String defaultConfigFile = configFiles[2];  // generic_logger.xml
 +    for (String f : configFiles) {
 +      if (new File(f).exists()) {
 +        return f;
 +      }
 +    }
 +    return defaultConfigFile;
 +  }
 +
 +  public static void setupLogging(String application) throws UnknownHostException {
 +    System.setProperty("org.apache.accumulo.core.application", application);
 +
 +    if (System.getenv("ACCUMULO_LOG_DIR") != null)
 +      System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_LOG_DIR"));
 +    else
 +      System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_HOME") + "/logs/");
 +
 +    String localhost = InetAddress.getLocalHost().getHostName();
 +    System.setProperty("org.apache.accumulo.core.ip.localhost.hostname", localhost);
 +
 +    // Use a specific log config, if it exists
 +    String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
 +    // Turn off messages about not being able to reach the remote logger... we protect against that.
 +    LogLog.setQuietMode(true);
 +
 +    // Read the auditing config
 +    String auditConfig = String.format("%s/auditLog.xml", System.getenv("ACCUMULO_CONF_DIR"));
 +
 +    // Set up local file-based logging right away
 +    Log4jConfiguration logConf = new Log4jConfiguration(logConfigFile);
 +    logConf.resetLogger();
 +
 +    // Watch the auditLog.xml for the future updates
 +    DOMConfigurator.configureAndWatch(auditConfig, 5000);
 +  }
 +
 +  public static void init(VolumeManager fs, ServerConfiguration serverConfig, String application) throws IOException {
 +    final AccumuloConfiguration conf = serverConfig.getConfiguration();
 +    final Instance instance = serverConfig.getInstance();
 +
 +    // Use a specific log config, if it exists
 +    final String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
 +
 +    // Set up polling log4j updates and log-forwarding using information advertised in zookeeper by the monitor
 +    MonitorLog4jWatcher logConfigWatcher = new MonitorLog4jWatcher(instance.getInstanceID(), logConfigFile);
 +    logConfigWatcher.setDelay(5000L);
 +    logConfigWatcher.start();
 +
 +    // Makes sure the log-forwarding to the monitor is configured
 +    int logPort = conf.getPort(Property.MONITOR_LOG4J_PORT);
 +    System.setProperty("org.apache.accumulo.core.host.log.port", Integer.toString(logPort));
 +
 +    log.info(application + " starting");
 +    log.info("Instance " + serverConfig.getInstance().getInstanceID());
 +    int dataVersion = Accumulo.getAccumuloPersistentVersion(fs);
 +    log.info("Data Version " + dataVersion);
 +    Accumulo.waitForZookeeperAndHdfs(fs);
 +
 +    Version codeVersion = new Version(Constants.VERSION);
 +    if (!(canUpgradeFromDataVersion(dataVersion))) {
 +      throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible with files stored using data version " + dataVersion);
 +    }
 +
 +    TreeMap<String,String> sortedProps = new TreeMap<String,String>();
 +    for (Entry<String,String> entry : conf)
 +      sortedProps.put(entry.getKey(), entry.getValue());
 +
 +    for (Entry<String,String> entry : sortedProps.entrySet()) {
 +      String key = entry.getKey();
 +      log.info(key + " = " + (Property.isSensitive(key) ? "<hidden>" : entry.getValue()));
 +    }
 +
 +    monitorSwappiness();
 +  }
 +
 +  /**
 +   * Sanity check that the current persistent version is allowed to upgrade to the version of Accumulo running.
 +   * @param dataVersion the version that is persisted in the backing Volumes
 +   */
 +  public static boolean canUpgradeFromDataVersion(final int dataVersion) {
 +    return dataVersion == ServerConstants.DATA_VERSION || dataVersion == ServerConstants.PREV_DATA_VERSION || dataVersion == ServerConstants.TWO_DATA_VERSIONS_AGO;
 +  }
 +
 +  /**
 +   * Does the data version number stored in the backing Volumes indicate we need to upgrade something?
 +   */
 +  public static boolean persistentVersionNeedsUpgrade(final int accumuloPersistentVersion) {
 +    return accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO || accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION;
 +  }
 +
 +  /**
 +   *
 +   */
 +  public static void monitorSwappiness() {
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        try {
 +          String procFile = "/proc/sys/vm/swappiness";
 +          File swappiness = new File(procFile);
 +          if (swappiness.exists() && swappiness.canRead()) {
 +            InputStream is = new FileInputStream(procFile);
 +            try {
 +              byte[] buffer = new byte[10];
 +              int bytes = is.read(buffer);
-               String setting = new String(buffer, 0, bytes, Constants.UTF8);
++              String setting = new String(buffer, 0, bytes, UTF_8);
 +              setting = setting.trim();
 +              if (bytes > 0 && Integer.parseInt(setting) > 10) {
 +                log.warn("System swappiness setting is greater than ten (" + setting + ") which can cause time-sensitive operations to be delayed. "
 +                    + " Accumulo is time sensitive because it needs to maintain distributed lock agreement.");
 +              }
 +            } finally {
 +              is.close();
 +            }
 +          }
 +        } catch (Throwable t) {
 +          log.error(t, t);
 +        }
 +      }
 +    }, 1000, 10 * 60 * 1000);
 +  }
 +
 +  public static void waitForZookeeperAndHdfs(VolumeManager fs) {
 +    log.info("Attempting to talk to zookeeper");
 +    while (true) {
 +      try {
 +        ZooReaderWriter.getInstance().getChildren(Constants.ZROOT);
 +        break;
 +      } catch (InterruptedException e) {
 +        // ignored
 +      } catch (KeeperException ex) {
 +        log.info("Waiting for accumulo to be initialized");
 +        UtilWaitThread.sleep(1000);
 +      }
 +    }
 +    log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
 +    long sleep = 1000;
 +    int unknownHostTries = 3;
 +    while (true) {
 +      try {
 +        if (fs.isReady())
 +          break;
 +        log.warn("Waiting for the NameNode to leave safemode");
 +      } catch (IOException ex) {
 +        log.warn("Unable to connect to HDFS", ex);
 +      } catch (IllegalArgumentException exception) {
 +        /* Unwrap the UnknownHostException so we can deal with it directly */
 +        if (exception.getCause() instanceof UnknownHostException) {
 +          if (unknownHostTries > 0) {
 +            log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause());
 +            /* We need to make sure our sleep period is long enough to avoid getting a cached failure of the host lookup. */
 +            sleep = Math.max(sleep, (AddressUtil.getAddressCacheNegativeTtl((UnknownHostException)(exception.getCause()))+1)*1000);
 +          } else {
 +            log.error("Unable to connect to HDFS and have exceeded max number of retries.", exception);
 +            throw exception;
 +          }
 +          unknownHostTries--;
 +        } else {
 +          throw exception;
 +        }
 +      }
 +      log.info("Backing off due to failure; current sleep period is " + sleep / 1000. + " seconds");
 +      UtilWaitThread.sleep(sleep);
 +      /* Back off to give transient failures more time to clear. */
 +      sleep = Math.min(60 * 1000, sleep * 2);
 +    }
 +    log.info("Connected to HDFS");
 +  }
 +
 +  /**
 +   * Exit loudly if there are outstanding Fate operations.
 +   * Since Fate serializes class names, we need to make sure there are no queued
 +   * transactions from a previous version before continuing an upgrade. The status of the operations is
 +   * irrelevant; those in SUCCESSFUL status cause the same problem as those just queued.
 +   *
 +   * Note that the Master should not allow write access to Fate until after all upgrade steps are complete.
 +   *
 +   * Should be called as a guard before performing any upgrade steps, after determining that an upgrade is needed.
 +   *
 +   * see ACCUMULO-2519
 +   */
 +  public static void abortIfFateTransactions() {
 +    try {
 +      final ReadOnlyTStore<Accumulo> fate = new ReadOnlyStore<Accumulo>(new ZooStore<Accumulo>(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZFATE,
 +          ZooReaderWriter.getInstance()));
 +      if (!(fate.list().isEmpty())) {
 +        throw new AccumuloException("Aborting upgrade because there are outstanding FATE transactions from a previous Accumulo version. Please see the README document for instructions on what to do under your previous version.");
 +      }
 +    } catch (Exception exception) {
 +      log.fatal("Problem verifying Fate readiness", exception);
 +      System.exit(1);
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 3508164,0000000..05f3c6c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -1,198 -1,0 +1,200 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.client;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.nio.ByteBuffer;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.impl.ConnectorImpl;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.util.ByteBufferUtil;
 +import org.apache.accumulo.core.util.OpTimer;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * An implementation of Instance that looks in HDFS and ZooKeeper to find the master and root tablet location.
 + * 
 + */
 +public class HdfsZooInstance implements Instance {
 +
 +  private HdfsZooInstance() {
 +    AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
 +    zooCache = new ZooCacheFactory().getZooCache(acuConf.get(Property.INSTANCE_ZK_HOST), (int) acuConf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
 +  }
 +
 +  private static HdfsZooInstance cachedHdfsZooInstance = null;
 +
 +  public static synchronized Instance getInstance() {
 +    if (cachedHdfsZooInstance == null)
 +      cachedHdfsZooInstance = new HdfsZooInstance();
 +    return cachedHdfsZooInstance;
 +  }
 +
 +  private static ZooCache zooCache;
 +  private static String instanceId = null;
 +  private static final Logger log = Logger.getLogger(HdfsZooInstance.class);
 +
 +  @Override
 +  public String getRootTabletLocation() {
 +    String zRootLocPath = ZooUtil.getRoot(this) + RootTable.ZROOT_TABLET_LOCATION;
 +
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zoocache.");
 +
 +    byte[] loc = zooCache.get(zRootLocPath);
 +
-     opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++    opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
 +
 +    if (loc == null) {
 +      return null;
 +    }
 +
-     return new String(loc, Constants.UTF8).split("\\|")[0];
++    return new String(loc, UTF_8).split("\\|")[0];
 +  }
 +
 +  @Override
 +  public List<String> getMasterLocations() {
 +
 +    String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
 +
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
 +
 +    byte[] loc = ZooLock.getLockData(zooCache, masterLocPath, null);
 +
-     opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++    opTimer.stop("Found master at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
 +
 +    if (loc == null) {
 +      return Collections.emptyList();
 +    }
 +
-     return Collections.singletonList(new String(loc, Constants.UTF8));
++    return Collections.singletonList(new String(loc, UTF_8));
 +  }
 +
 +  @Override
 +  public String getInstanceID() {
 +    if (instanceId == null)
 +      _getInstanceID();
 +    return instanceId;
 +  }
 +
 +  private static synchronized void _getInstanceID() {
 +    if (instanceId == null) {
 +      AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
 +      // InstanceID should be the same across all volumes, so just choose one
 +      VolumeManager fs;
 +      try {
 +        fs = VolumeManagerImpl.get();
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
 +      log.trace("Looking for instanceId from " + instanceIdPath);
 +      String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
 +      instanceId = instanceIdFromFile;
 +    }
 +  }
 +
 +  @Override
 +  public String getInstanceName() {
 +    return ZooKeeperInstance.lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
 +  }
 +
 +  @Override
 +  public String getZooKeepers() {
 +    return ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
 +  }
 +
 +  @Override
 +  public int getZooKeepersSessionTimeOut() {
 +    return (int) ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
 +  }
 +
 +  @Override
 +  public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
 +    return new ConnectorImpl(this, new Credentials(principal, token));
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
 +    return getConnector(user, new PasswordToken(pass));
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
 +    return getConnector(user, ByteBufferUtil.toBytes(pass));
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
 +    return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
 +  }
 +
 +  private AccumuloConfiguration conf = null;
 +
 +  @Deprecated
 +  @Override
 +  public AccumuloConfiguration getConfiguration() {
 +    if (conf == null)
 +      conf = new ServerConfiguration(this).getConfiguration();
 +    return conf;
 +  }
 +
 +  @Override
 +  @Deprecated
 +  public void setConfiguration(AccumuloConfiguration conf) {
 +    this.conf = conf;
 +  }
 +
 +  public static void main(String[] args) {
 +    Instance instance = HdfsZooInstance.getInstance();
 +    System.out.println("Instance Name: " + instance.getInstanceName());
 +    System.out.println("Instance ID: " + instance.getInstanceID());
 +    System.out.println("ZooKeepers: " + instance.getZooKeepers());
 +    System.out.println("Masters: " + StringUtil.join(instance.getMasterLocations(), ", "));
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
index df1be50,0000000..18feb4f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
@@@ -1,160 -1,0 +1,161 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.List;
 +import java.util.Map;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * A helper object for accessing properties in a {@link ZooCache}.
 + */
 +public class ZooCachePropertyAccessor {
 +  private static final Logger log = Logger.getLogger(ZooCachePropertyAccessor.class);
 +
 +  static class PropCacheKey {
 +    final String instanceId;
 +    final String scope;
 +
 +    PropCacheKey(String iid, String s) {
 +      instanceId = iid;
 +      scope = s;
 +    }
 +
 +    @Override
 +    public boolean equals(Object other) {
 +      if (this == other) {
 +        return true;
 +      }
 +      if (!(other instanceof PropCacheKey)) {
 +        return false;
 +      }
 +      PropCacheKey o = (PropCacheKey) other;
 +      return (instanceId.equals(o.instanceId) && scope.equals(o.scope));
 +    }
 +
 +    @Override
 +    public int hashCode() {
 +      int c = 17;
 +      c = (37 * c) + instanceId.hashCode();
 +      c = (37 * c) + scope.hashCode();
 +      return c;
 +    }
 +  }
 +
 +  private final ZooCache propCache;
 +
 +  /**
 +   * Creates a new accessor.
 +   *
 +   * @param propCache
 +   *          property cache
 +   */
 +  ZooCachePropertyAccessor(ZooCache propCache) {
 +    this.propCache = propCache;
 +  }
 +
 +  /**
 +   * Gets the property cache accessed by this object.
 +   *
 +   * @return property cache
 +   */
 +  ZooCache getZooCache() {
 +    return propCache;
 +  }
 +
 +  /**
 +   * Gets a property. If the property is not in ZooKeeper or is present but an invalid format for the property type, the parent configuration is consulted (if
 +   * provided).
 +   *
 +   * @param property
 +   *          property to get
 +   * @param path
 +   *          ZooKeeper path where properties lie
 +   * @param parent
 +   *          parent configuration (optional)
 +   * @return property value, or null if not found
 +   */
 +  String get(Property property, String path, AccumuloConfiguration parent) {
 +    String key = property.getKey();
 +    String value = get(path + "/" + key);
 +
 +    if (value == null || !property.getType().isValidFormat(value)) {
 +      if (value != null) {
 +        log.error("Using default value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
 +      }
 +      if (parent != null) {
 +        value = parent.get(property);
 +      }
 +    }
 +    return value;
 +  }
 +
 +  private String get(String path) {
 +    byte[] v = propCache.get(path);
 +    if (v != null) {
-       return new String(v, Constants.UTF8);
++      return new String(v, UTF_8);
 +    } else {
 +      return null;
 +    }
 +  }
 +
 +  /**
 +   * Gets all properties into the given map. Properties are filtered using the given filter. Properties from a parent configuration are also added to the map
 +   * and filtered, either using a separate filter or, if not specified, the other filter.
 +   *
 +   * @param props
 +   *          map to populate with properties
 +   * @param path
 +   *          ZooKeeper path where properties lie
 +   * @param filter
 +   *          property filter (required)
 +   * @param parent
 +   *          parent configuration (required)
 +   * @param parentFilter
 +   *          separate filter for parent properties (optional)
 +   */
 +  void getProperties(Map<String,String> props, String path, PropertyFilter filter, AccumuloConfiguration parent, PropertyFilter parentFilter) {
 +    parent.getProperties(props, parentFilter != null ? parentFilter : filter);
 +
 +    List<String> children = propCache.getChildren(path);
 +    if (children != null) {
 +      for (String child : children) {
 +        if (child != null && filter.accept(child)) {
 +          String value = get(path + "/" + child);
 +          if (value != null) {
 +            props.put(child, value);
 +          }
 +        }
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Clears the internal {@link ZooCache}.
 +   */
 +  void invalidateCache() {
 +    propCache.clear();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index a49ec43,0000000..6d25f1d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@@ -1,118 -1,0 +1,120 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.conf;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.log4j.Logger;
 +
 +public class ZooConfiguration extends AccumuloConfiguration {
 +  private static final Logger log = Logger.getLogger(ZooConfiguration.class);
 +
 +  private final String instanceId;
 +  private final ZooCache propCache;
 +  private final AccumuloConfiguration parent;
 +  private final Map<String,String> fixedProps = Collections.synchronizedMap(new HashMap<String,String>());
 +
 +  protected ZooConfiguration(String instanceId, ZooCache propCache, AccumuloConfiguration parent) {
 +    this.instanceId = instanceId;
 +    this.propCache = propCache;
 +    this.parent = parent;
 +  }
 +
 +  @Override
 +  public void invalidateCache() {
 +    if (propCache != null)
 +      propCache.clear();
 +  }
 +
 +  /**
 +   * Gets the parent configuration of this configuration.
 +   *
 +   * @return parent configuration
 +   */
 +  public AccumuloConfiguration getParentConfiguration() {
 +    return parent;
 +  }
 +
 +  private String _get(Property property) {
 +    String key = property.getKey();
 +    String value = null;
 +
 +    if (Property.isValidZooPropertyKey(key)) {
 +      value = get(key);
 +    }
 +
 +    if (value == null || !property.getType().isValidFormat(value)) {
 +      if (value != null)
 +        log.error("Using parent value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
 +      value = parent.get(property);
 +    }
 +    return value;
 +  }
 +
 +  @Override
 +  public String get(Property property) {
 +    if (Property.isFixedZooPropertyKey(property)) {
 +      if (fixedProps.containsKey(property.getKey())) {
 +        return fixedProps.get(property.getKey());
 +      } else {
 +        synchronized (fixedProps) {
 +          String val = _get(property);
 +          fixedProps.put(property.getKey(), val);
 +          return val;
 +        }
 +
 +      }
 +    } else {
 +      return _get(property);
 +    }
 +  }
 +
 +  private String get(String key) {
 +    String zPath = ZooUtil.getRoot(instanceId) + Constants.ZCONFIG + "/" + key;
 +    byte[] v = propCache.get(zPath);
 +    String value = null;
 +    if (v != null)
-       value = new String(v, Constants.UTF8);
++      value = new String(v, UTF_8);
 +    return value;
 +  }
 +
 +  @Override
 +  public void getProperties(Map<String,String> props, PropertyFilter filter) {
 +    parent.getProperties(props, filter);
 +
 +    List<String> children = propCache.getChildren(ZooUtil.getRoot(instanceId) + Constants.ZCONFIG);
 +    if (children != null) {
 +      for (String child : children) {
 +        if (child != null && filter.accept(child)) {
 +          String value = get(child);
 +          if (value != null)
 +            props.put(child, value);
 +        }
 +      }
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 807e821,0000000..587e38a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -1,315 -1,0 +1,317 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.constraints;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.HashSet;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.constraints.Constraint;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +public class MetadataConstraints implements Constraint {
 +  
 +  private ZooCache zooCache = null;
 +  private String zooRoot = null;
 +  
 +  private static final Logger log = Logger.getLogger(MetadataConstraints.class);
 +  
 +  private static boolean[] validTableNameChars = new boolean[256];
 +  
 +  {
 +    for (int i = 0; i < 256; i++) {
 +      validTableNameChars[i] = ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) || i == '!';
 +    }
 +  }
 +  
 +  private static final HashSet<ColumnFQ> validColumnQuals = new HashSet<ColumnFQ>(Arrays.asList(new ColumnFQ[] {
 +      TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN,
 +      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN,
 +      TabletsSection.ServerColumnFamily.LOCK_COLUMN, TabletsSection.ServerColumnFamily.FLUSH_COLUMN, TabletsSection.ServerColumnFamily.COMPACT_COLUMN}));
 +  
 +  private static final HashSet<Text> validColumnFams = new HashSet<Text>(Arrays.asList(new Text[] {TabletsSection.BulkFileColumnFamily.NAME,
 +      LogColumnFamily.NAME, ScanFileColumnFamily.NAME, DataFileColumnFamily.NAME,
 +      TabletsSection.CurrentLocationColumnFamily.NAME, TabletsSection.LastLocationColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME,
 +      ChoppedColumnFamily.NAME, ClonedColumnFamily.NAME}));
 +  
 +  private static boolean isValidColumn(ColumnUpdate cu) {
 +    
 +    if (validColumnFams.contains(new Text(cu.getColumnFamily())))
 +      return true;
 +    
 +    if (validColumnQuals.contains(new ColumnFQ(cu)))
 +      return true;
 +    
 +    return false;
 +  }
 +  
 +  static private ArrayList<Short> addViolation(ArrayList<Short> lst, int violation) {
 +    if (lst == null)
 +      lst = new ArrayList<Short>();
 +    lst.add((short) violation);
 +    return lst;
 +  }
 +  
 +  static private ArrayList<Short> addIfNotPresent(ArrayList<Short> lst, int intViolation) {
 +    if (lst == null)
 +      return addViolation(lst, intViolation);
 +    short violation = (short) intViolation;
 +    if (!lst.contains(violation))
 +      return addViolation(lst, intViolation);
 +    return lst;
 +  }
 +  
 +  @Override
 +  public List<Short> check(Environment env, Mutation mutation) {
 +    
 +    ArrayList<Short> violations = null;
 +    
 +    Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
 +    
 +    // check the row, it should contains at least one ; or end with <
 +    boolean containsSemiC = false;
 +    
 +    byte[] row = mutation.getRow();
 +    
 +    // always allow rows that fall within reserved areas
 +    if (row.length > 0 && row[0] == '~')
 +      return null;
 +    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
 +      return null;
 +    
 +    for (byte b : row) {
 +      if (b == ';') {
 +        containsSemiC = true;
 +      }
 +      
 +      if (b == ';' || b == '<')
 +        break;
 +      
 +      if (!validTableNameChars[0xff & b]) {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    }
 +    
 +    if (!containsSemiC) {
 +      // see if last row char is <
 +      if (row.length == 0 || row[row.length - 1] != '<') {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    } else {
 +      if (row.length == 0) {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    }
 +    
 +    if (row.length > 0 && row[0] == '!') {
 +      if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
 +        violations = addIfNotPresent(violations, 4);
 +      }
 +    }
 +    
 +    // ensure row is not less than Constants.METADATA_TABLE_ID
 +    if (new Text(row).compareTo(new Text(MetadataTable.ID)) < 0) {
 +      violations = addViolation(violations, 5);
 +    }
 +    
 +    boolean checkedBulk = false;
 +    
 +    for (ColumnUpdate columnUpdate : colUpdates) {
 +      Text columnFamily = new Text(columnUpdate.getColumnFamily());
 +      
 +      if (columnUpdate.isDeleted()) {
 +        if (!isValidColumn(columnUpdate)) {
 +          violations = addViolation(violations, 2);
 +        }
 +        continue;
 +      }
 +      
 +      if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
 +        violations = addViolation(violations, 6);
 +      }
 +      
 +      if (columnFamily.equals(DataFileColumnFamily.NAME)) {
 +        try {
 +          DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
 +          
 +          if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
 +            violations = addViolation(violations, 1);
 +          }
 +        } catch (NumberFormatException nfe) {
 +          violations = addViolation(violations, 1);
 +        } catch (ArrayIndexOutOfBoundsException aiooe) {
 +          violations = addViolation(violations, 1);
 +        }
 +      } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
 +        
 +      } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
 +        if (!columnUpdate.isDeleted() && !checkedBulk) {
 +          // splits, which also write the time reference, are allowed to write this reference even when
 +          // the transaction is not running because the other half of the tablet is holding a reference
 +          // to the file.
 +          boolean isSplitMutation = false;
 +          // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
 +          // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
 +          // See ACCUMULO-1230.
 +          boolean isLocationMutation = false;
 +          
 +          HashSet<Text> dataFiles = new HashSet<Text>();
 +          HashSet<Text> loadedFiles = new HashSet<Text>();
 +
-           String tidString = new String(columnUpdate.getValue(), Constants.UTF8);
++          String tidString = new String(columnUpdate.getValue(), UTF_8);
 +          int otherTidCount = 0;
 +          
 +          for (ColumnUpdate update : mutation.getUpdates()) {
 +            if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
 +              isSplitMutation = true;
 +            } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
 +              isLocationMutation = true;
 +            } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
 +              dataFiles.add(new Text(update.getColumnQualifier()));
 +            } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
 +              loadedFiles.add(new Text(update.getColumnQualifier()));
 +              
-               if (!new String(update.getValue(), Constants.UTF8).equals(tidString)) {
++              if (!new String(update.getValue(), UTF_8).equals(tidString)) {
 +                otherTidCount++;
 +              }
 +            }
 +          }
 +          
 +          if (!isSplitMutation && !isLocationMutation) {
 +            long tid = Long.parseLong(tidString);
 +            
 +            try {
 +              if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
 +                violations = addViolation(violations, 8);
 +              }
 +            } catch (Exception ex) {
 +              violations = addViolation(violations, 8);
 +            }
 +          }
 +          
 +          checkedBulk = true;
 +        }
 +      } else {
 +        if (!isValidColumn(columnUpdate)) {
 +          violations = addViolation(violations, 2);
 +        } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0
 +            && (violations == null || !violations.contains((short) 4))) {
 +          KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
 +          
 +          Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
 +          
 +          boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
 +          
 +          if (!prevEndRowLessThanEndRow) {
 +            violations = addViolation(violations, 3);
 +          }
 +        } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
 +          if (zooCache == null) {
 +            zooCache = new ZooCache();
 +          }
 +          
 +          if (zooRoot == null) {
 +            zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
 +          }
 +          
 +          boolean lockHeld = false;
-           String lockId = new String(columnUpdate.getValue(), Constants.UTF8);
++          String lockId = new String(columnUpdate.getValue(), UTF_8);
 +          
 +          try {
 +            lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
 +          } catch (Exception e) {
 +            log.debug("Failed to verify lock was held " + lockId + " " + e.getMessage());
 +          }
 +          
 +          if (!lockHeld) {
 +            violations = addViolation(violations, 7);
 +          }
 +        }
 +        
 +      }
 +    }
 +    
 +    if (violations != null) {
-       log.debug("violating metadata mutation : " + new String(mutation.getRow(), Constants.UTF8));
++      log.debug("violating metadata mutation : " + new String(mutation.getRow(), UTF_8));
 +      for (ColumnUpdate update : mutation.getUpdates()) {
-         log.debug(" update: " + new String(update.getColumnFamily(), Constants.UTF8) + ":" + new String(update.getColumnQualifier(), Constants.UTF8) + " value "
-             + (update.isDeleted() ? "[delete]" : new String(update.getValue(), Constants.UTF8)));
++        log.debug(" update: " + new String(update.getColumnFamily(), UTF_8) + ":" + new String(update.getColumnQualifier(), UTF_8) + " value "
++            + (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
 +      }
 +    }
 +    
 +    return violations;
 +  }
 +  
 +  protected Arbitrator getArbitrator() {
 +    return new ZooArbitrator();
 +  }
 +  
 +  @Override
 +  public String getViolationDescription(short violationCode) {
 +    switch (violationCode) {
 +      case 1:
 +        return "data file size must be a non-negative integer";
 +      case 2:
 +        return "Invalid column name given.";
 +      case 3:
 +        return "Prev end row is greater than or equal to end row.";
 +      case 4:
 +        return "Invalid metadata row format";
 +      case 5:
 +        return "Row can not be less than " + MetadataTable.ID;
 +      case 6:
 +        return "Empty values are not allowed for any " + MetadataTable.NAME + " column";
 +      case 7:
 +        return "Lock not held in zookeeper by writer";
 +      case 8:
 +        return "Bulk load transaction no longer running";
 +    }
 +    return null;
 +  }
 +  
 +  @Override
 +  protected void finalize() {
 +    if (zooCache != null)
 +      zooCache.clear();
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 24b5605,0000000..7c4b8d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@@ -1,586 -1,0 +1,587 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.init;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.util.Arrays;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Locale;
 +import java.util.Map.Entry;
 +import java.util.UUID;
 +
 +import jline.console.ConsoleReader;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVWriter;
 +import org.apache.accumulo.core.iterators.user.VersioningIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.master.thrift.MasterGoalState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.constraints.MetadataConstraints;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.util.TablePropUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.ZooDefs.Ids;
 +
 +import com.beust.jcommander.Parameter;
 +
 +/**
 + * This class is used to setup the directory structure and the root tablet to get an instance started
 + *
 + */
 +public class Initialize {
 +  private static final Logger log = Logger.getLogger(Initialize.class);
 +  private static final String DEFAULT_ROOT_USER = "root";
 +  public static final String TABLE_TABLETS_TABLET_DIR = "/table_info";
 +
 +  private static ConsoleReader reader = null;
 +  private static IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +  private static ConsoleReader getConsoleReader() throws IOException {
 +    if (reader == null)
 +      reader = new ConsoleReader();
 +    return reader;
 +  }
 +
 +  /**
 +   * Sets this class's ZooKeeper reader/writer.
 +   *
 +   * @param izoo
 +   *          reader/writer
 +   */
 +  static void setZooReaderWriter(IZooReaderWriter izoo) {
 +    zoo = izoo;
 +  }
 +
 +  /**
 +   * Gets this class's ZooKeeper reader/writer.
 +   *
 +   * @return reader/writer
 +   */
 +  static IZooReaderWriter getZooReaderWriter() {
 +    return zoo;
 +  }
 +
 +  private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
 +  static {
 +    initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K");
 +    initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
 +    initialMetadataConf.put(Property.TABLE_WALOG_ENABLED.getKey(), "true");
 +    initialMetadataConf.put(Property.TABLE_MAJC_RATIO.getKey(), "1");
 +    initialMetadataConf.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "64M");
 +    initialMetadataConf.put(Property.TABLE_CONSTRAINT_PREFIX.getKey() + "1", MetadataConstraints.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers", "10," + VersioningIterator.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers.opt.maxVersions", "1");
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers", "10," + VersioningIterator.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers.opt.maxVersions", "1");
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers", "10," + VersioningIterator.class.getName());
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers.opt.maxVersions", "1");
 +    initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.bulkLoadFilter", "20," + MetadataBulkLoadFilter.class.getName());
 +    initialMetadataConf.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
 +    initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
 +        String.format("%s,%s", TabletsSection.TabletColumnFamily.NAME, TabletsSection.CurrentLocationColumnFamily.NAME));
 +    initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s", TabletsSection.DataFileColumnFamily.NAME,
 +        TabletsSection.LogColumnFamily.NAME, TabletsSection.ServerColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME));
 +    initialMetadataConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
 +    initialMetadataConf.put(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
 +    initialMetadataConf.put(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
 +    initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
 +  }
 +
 +  static boolean checkInit(Configuration conf, VolumeManager fs, SiteConfiguration sconf) throws IOException {
 +    @SuppressWarnings("deprecation")
 +    String fsUri = sconf.get(Property.INSTANCE_DFS_URI);
 +    if (fsUri.equals(""))
 +      fsUri = FileSystem.getDefaultUri(conf).toString();
 +    log.info("Hadoop Filesystem is " + fsUri);
 +    log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
 +    log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
 +    log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
 +    if (!zookeeperAvailable()) {
 +      log.fatal("Zookeeper needs to be up and running in order to init. Exiting ...");
 +      return false;
 +    }
 +    if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) {
 +      ConsoleReader c = getConsoleReader();
 +      c.beep();
 +      c.println();
 +      c.println();
 +      c.println("Warning!!! Your instance secret is still set to the default, this is not secure. We highly recommend you change it.");
 +      c.println();
 +      c.println();
 +      c.println("You can change the instance secret in accumulo by using:");
 +      c.println("   bin/accumulo " + org.apache.accumulo.server.util.ChangeSecret.class.getName() + " oldPassword newPassword.");
 +      c.println("You will also need to edit your secret in your configuration file by adding the property instance.secret to your conf/accumulo-site.xml. Without this accumulo will not operate correctly");
 +    }
 +    try {
 +      if (isInitialized(fs)) {
 +        printInitializeFailureMessages(sconf);
 +        return false;
 +      }
 +    } catch (IOException e) {
 +      throw new IOException("Failed to check if filesystem already initialized", e);
 +    }
 +
 +    return true;
 +  }
 +
 +  static void printInitializeFailureMessages(SiteConfiguration sconf) {
 +    @SuppressWarnings("deprecation")
 +    Property INSTANCE_DFS_DIR = Property.INSTANCE_DFS_DIR;
 +    @SuppressWarnings("deprecation")
 +    Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI;
 +    String instanceDfsDir = sconf.get(INSTANCE_DFS_DIR);
 +    log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()))
 +        + " were previously initialized.");
 +    String instanceVolumes = sconf.get(Property.INSTANCE_VOLUMES);
 +    String instanceDfsUri = sconf.get(INSTANCE_DFS_URI);
 +
 +    if (!instanceVolumes.isEmpty()) {
 +      log.fatal("Change the property " + Property.INSTANCE_VOLUMES + " to use different filesystems,");
 +    } else if (!instanceDfsDir.isEmpty()) {
 +      log.fatal("Change the property " + INSTANCE_DFS_URI + " to use a different filesystem,");
 +    } else {
 +      log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_VOLUMES + " to use a different filesystem,");
 +    }
 +    log.fatal("or change the property " + INSTANCE_DFS_DIR + " to use a different directory.");
 +    log.fatal("The current value of " + INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|");
 +    log.fatal("The current value of " + INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|");
 +    log.fatal("The current value of " + Property.INSTANCE_VOLUMES + " is |" + instanceVolumes + "|");
 +  }
 +
 +  public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException {
 +    if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) {
 +      return false;
 +    }
 +
 +    // prompt user for instance name and root password early, in case they
 +    // abort, we don't leave an inconsistent HDFS/ZooKeeper structure
 +    String instanceNamePath;
 +    try {
 +      instanceNamePath = getInstanceNamePath(opts);
 +    } catch (Exception e) {
 +      log.fatal("Failed to talk to zookeeper", e);
 +      return false;
 +    }
 +    opts.rootpass = getRootPassword(opts);
 +    return initialize(opts, instanceNamePath, fs);
 +  }
 +
 +  public static boolean initialize(Opts opts, String instanceNamePath, VolumeManager fs) {
 +
 +    UUID uuid = UUID.randomUUID();
 +    // the actual disk locations of the root table and tablets
 +    String[] configuredVolumes = VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration());
 +    final String rootTabletDir = new Path(fs.choose(configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + RootTable.ID
 +        + RootTable.ROOT_TABLET_LOCATION).toString();
 +
 +    try {
 +      initZooKeeper(opts, uuid.toString(), instanceNamePath, rootTabletDir);
 +    } catch (Exception e) {
 +      log.fatal("Failed to initialize zookeeper", e);
 +      return false;
 +    }
 +
 +    try {
 +      initFileSystem(opts, fs, uuid, rootTabletDir);
 +    } catch (Exception e) {
 +      log.fatal("Failed to initialize filesystem", e);
 +
 +      if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES).trim().equals("")) {
 +        Configuration fsConf = CachedConfiguration.getInstance();
 +
 +        final String defaultFsUri = "file:///";
 +        String fsDefaultName = fsConf.get("fs.default.name", defaultFsUri), fsDefaultFS = fsConf.get("fs.defaultFS", defaultFsUri);
 +
 +        // Try to determine when we couldn't find an appropriate core-site.xml on the classpath
 +        if (defaultFsUri.equals(fsDefaultName) && defaultFsUri.equals(fsDefaultFS)) {
 +          log.fatal("Default filesystem value ('fs.defaultFS' or 'fs.default.name') of '" + defaultFsUri + "' was found in the Hadoop configuration");
 +          log.fatal("Please ensure that the Hadoop core-site.xml is on the classpath using 'general.classpaths' in accumulo-site.xml");
 +        }
 +      }
 +
 +      return false;
 +    }
 +
 +    try {
 +      initSecurity(opts, uuid.toString());
 +    } catch (Exception e) {
 +      log.fatal("Failed to initialize security", e);
 +      return false;
 +    }
 +    return true;
 +  }
 +
 +  private static boolean zookeeperAvailable() {
 +    try {
 +      return zoo.exists("/");
 +    } catch (KeeperException e) {
 +      return false;
 +    } catch (InterruptedException e) {
 +      return false;
 +    }
 +  }
 +
 +  private static void initDirs(VolumeManager fs, UUID uuid, String[] baseDirs, boolean print) throws IOException {
 +    for (String baseDir : baseDirs) {
 +      fs.mkdirs(new Path(new Path(baseDir, ServerConstants.VERSION_DIR), "" + ServerConstants.DATA_VERSION));
 +
 +      // create an instance id
 +      Path iidLocation = new Path(baseDir, ServerConstants.INSTANCE_ID_DIR);
 +      fs.mkdirs(iidLocation);
 +      fs.createNewFile(new Path(iidLocation, uuid.toString()));
 +      if (print)
 +        log.info("Initialized volume " + baseDir);
 +    }
 +  }
 +
 +  private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, String rootTabletDir) throws IOException {
 +    initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
 +
 +    // initialize initial metadata config in zookeeper
 +    initMetadataConfig();
 +
 +    String tableMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
 +        + TABLE_TABLETS_TABLET_DIR;
 +    String defaultMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
 +        + Constants.DEFAULT_TABLET_LOCATION;
 +
 +    // create table and default tablets directories
 +    createDirectories(fs, rootTabletDir, tableMetadataTabletDir, defaultMetadataTabletDir);
 +
 +    // populate the root tablet with info about the metadata tablets
 +    String fileName = rootTabletDir + Path.SEPARATOR + "00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
 +    createMetadataFile(fs, fileName, MetadataTable.ID, tableMetadataTabletDir, defaultMetadataTabletDir);
 +  }
 +
 +  /**
 +   * Create an rfile in the default tablet's directory for a new table. This method is used to create the initial root tablet contents, with information about
 +   * the metadata table's tablets
 +   *
 +   * @param volmanager
 +   *          The VolumeManager
 +   * @param fileName
 +   *          The location to create the file
 +   * @param tableId
 +   *          TableID that is being "created"
 +   * @param tableTabletDir
 +   *          The table_info directory for the new table
 +   * @param defaultTabletDir
 +   *          The default_tablet directory for the new table
 +   */
 +  private static void createMetadataFile(VolumeManager volmanager, String fileName, String tableId, String tableTabletDir, String defaultTabletDir)
 +      throws IOException {
 +    FileSystem fs = volmanager.getVolumeByPath(new Path(fileName)).getFileSystem();
 +    FileSKVWriter tabletWriter = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
 +    tabletWriter.startDefaultLocalityGroup();
 +
 +    Text splitPoint = TabletsSection.getRange().getEndKey().getRow();
 +    createEntriesForTablet(tabletWriter, tableId, tableTabletDir, null, splitPoint);
 +    createEntriesForTablet(tabletWriter, tableId, defaultTabletDir, splitPoint, null);
 +
 +    tabletWriter.close();
 +  }
 +
 +  private static void createEntriesForTablet(FileSKVWriter writer, String tableId, String tabletDir, Text tabletPrevEndRow, Text tabletEndRow)
 +      throws IOException {
 +    Text extent = new Text(KeyExtent.getMetadataEntry(new Text(tableId), tabletEndRow));
-     addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(Constants.UTF8)));
-     addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(Constants.UTF8)));
++    addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(UTF_8)));
++    addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8)));
 +    addEntry(writer, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tabletPrevEndRow));
 +  }
 +
 +  private static void addEntry(FileSKVWriter writer, Text row, ColumnFQ col, Value value) throws IOException {
 +    writer.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier(), 0), value);
 +  }
 +
 +  private static void createDirectories(VolumeManager fs, String... dirs) throws IOException {
 +    for (String s : dirs) {
 +      Path dir = new Path(s);
 +      try {
 +        FileStatus fstat = fs.getFileStatus(dir);
 +        // TODO Remove deprecation warning suppression when Hadoop1 support is dropped
 +        @SuppressWarnings("deprecation")
 +        boolean isDirectory = fstat.isDir();
 +        if (!isDirectory) {
 +          log.fatal("location " + dir + " exists but is not a directory");
 +          return;
 +        }
 +      } catch (FileNotFoundException fnfe) {
 +        // attempt to create directory, since it doesn't exist
 +        if (!fs.mkdirs(dir)) {
 +          log.fatal("unable to create directory " + dir);
 +          return;
 +        }
 +      }
 +    }
 +  }
 +
 +  private static void initZooKeeper(Opts opts, String uuid, String instanceNamePath, String rootTabletDir) throws KeeperException, InterruptedException {
 +    // setup basic data in zookeeper
 +    zoo.putPersistentData(Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
 +    zoo.putPersistentData(Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
 +
 +    // setup instance name
 +    if (opts.clearInstanceName)
 +      zoo.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
-     zoo.putPersistentData(instanceNamePath, uuid.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(instanceNamePath, uuid.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +
 +    final byte[] EMPTY_BYTE_ARRAY = new byte[0], ZERO_CHAR_ARRAY = new byte[] {'0'};
 +
 +    // setup the instance
 +    String zkInstanceRoot = Constants.ZROOT + "/" + uuid;
 +    zoo.putPersistentData(zkInstanceRoot, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES, Constants.ZTABLES_INITIAL_ID, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES, new byte[0], NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewNamespaceState(uuid, Namespaces.DEFAULT_NAMESPACE_ID, Namespaces.DEFAULT_NAMESPACE, NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewNamespaceState(uuid, Namespaces.ACCUMULO_NAMESPACE_ID, Namespaces.ACCUMULO_NAMESPACE, NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewTableState(uuid, RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
 +    TableManager.prepareNewTableState(uuid, MetadataTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, MetadataTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTSERVERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZPROBLEMS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_WALOGS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
-     zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
-     zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8), NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZNEXT_FILE, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZRECOVERY, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +    zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
 +  }
 +
 +  private static String getInstanceNamePath(Opts opts) throws IOException, KeeperException, InterruptedException {
 +    // setup the instance name
 +    String instanceName, instanceNamePath = null;
 +    boolean exists = true;
 +    do {
 +      if (opts.cliInstanceName == null) {
 +        instanceName = getConsoleReader().readLine("Instance name : ");
 +      } else {
 +        instanceName = opts.cliInstanceName;
 +      }
 +      if (instanceName == null)
 +        System.exit(0);
 +      instanceName = instanceName.trim();
 +      if (instanceName.length() == 0)
 +        continue;
 +      instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
 +      if (opts.clearInstanceName) {
 +        exists = false;
 +        break;
 +      } else if (exists = zoo.exists(instanceNamePath)) {
 +        String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : ");
 +        if (decision == null)
 +          System.exit(0);
 +        if (decision.length() == 1 && decision.toLowerCase(Locale.ENGLISH).charAt(0) == 'y') {
 +          opts.clearInstanceName = true;
 +          exists = false;
 +        }
 +      }
 +    } while (exists);
 +    return instanceNamePath;
 +  }
 +
 +  private static byte[] getRootPassword(Opts opts) throws IOException {
 +    if (opts.cliPassword != null) {
-       return opts.cliPassword.getBytes(Constants.UTF8);
++      return opts.cliPassword.getBytes(UTF_8);
 +    }
 +    String rootpass;
 +    String confirmpass;
 +    do {
 +      rootpass = getConsoleReader()
 +          .readLine("Enter initial password for " + DEFAULT_ROOT_USER + " (this may not be applicable for your security setup): ", '*');
 +      if (rootpass == null)
 +        System.exit(0);
 +      confirmpass = getConsoleReader().readLine("Confirm initial password for " + DEFAULT_ROOT_USER + ": ", '*');
 +      if (confirmpass == null)
 +        System.exit(0);
 +      if (!rootpass.equals(confirmpass))
 +        log.error("Passwords do not match");
 +    } while (!rootpass.equals(confirmpass));
-     return rootpass.getBytes(Constants.UTF8);
++    return rootpass.getBytes(UTF_8);
 +  }
 +
 +  private static void initSecurity(Opts opts, String iid) throws AccumuloSecurityException, ThriftSecurityException {
 +    AuditedSecurityOperation.getInstance(iid, true).initializeSecurity(SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()), DEFAULT_ROOT_USER,
 +        opts.rootpass);
 +  }
 +
 +  public static void initMetadataConfig() throws IOException {
 +    try {
 +      Configuration conf = CachedConfiguration.getInstance();
 +      int max = conf.getInt("dfs.replication.max", 512);
 +      // Hadoop 0.23 switched the min value configuration name
 +      int min = Math.max(conf.getInt("dfs.replication.min", 1), conf.getInt("dfs.namenode.replication.min", 1));
 +      if (max < 5)
 +        setMetadataReplication(max, "max");
 +      if (min > 5)
 +        setMetadataReplication(min, "min");
 +      for (Entry<String,String> entry : initialMetadataConf.entrySet()) {
 +        if (!TablePropUtil.setTableProperty(RootTable.ID, entry.getKey(), entry.getValue()))
 +          throw new IOException("Cannot create per-table property " + entry.getKey());
 +        if (!TablePropUtil.setTableProperty(MetadataTable.ID, entry.getKey(), entry.getValue()))
 +          throw new IOException("Cannot create per-table property " + entry.getKey());
 +      }
 +    } catch (Exception e) {
 +      log.fatal("error talking to zookeeper", e);
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  private static void setMetadataReplication(int replication, String reason) throws IOException {
 +    String rep = getConsoleReader().readLine(
 +        "Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME + " replication of 5. What do you want to set your "
 +            + MetadataTable.NAME + " replication to? (" + replication + ") ");
 +    if (rep == null || rep.length() == 0)
 +      rep = Integer.toString(replication);
 +    else
 +      // Lets make sure it's a number
 +      Integer.parseInt(rep);
 +    initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), rep);
 +  }
 +
 +  public static boolean isInitialized(VolumeManager fs) throws IOException {
 +    for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
 +      if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR)))
 +        return true;
 +    }
 +
 +    return false;
 +  }
 +
 +  private static void addVolumes(VolumeManager fs) throws IOException {
 +    HashSet<String> initializedDirs = new HashSet<String>();
 +    initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
 +
 +    HashSet<String> uinitializedDirs = new HashSet<String>();
 +    uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
 +    uinitializedDirs.removeAll(initializedDirs);
 +
 +    Path aBasePath = new Path(initializedDirs.iterator().next());
 +    Path iidPath = new Path(aBasePath, ServerConstants.INSTANCE_ID_DIR);
 +    Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
 +
 +    UUID uuid = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(iidPath, ServerConfiguration.getSiteConfiguration()));
 +
 +    if (ServerConstants.DATA_VERSION != Accumulo.getAccumuloPersistentVersion(versionPath.getFileSystem(CachedConfiguration.getInstance()), versionPath)) {
 +      throw new IOException("Accumulo " + Constants.VERSION + " cannot initialize data version " + Accumulo.getAccumuloPersistentVersion(fs));
 +    }
 +
 +    initDirs(fs, uuid, uinitializedDirs.toArray(new String[uinitializedDirs.size()]), true);
 +  }
 +
 +  static class Opts extends Help {
 +    @Parameter(names = "--add-volumes", description = "Initialize any uninitialized volumes listed in instance.volumes")
 +    boolean addVolumes = false;
 +    @Parameter(names = "--reset-security", description = "just update the security information")
 +    boolean resetSecurity = false;
 +    @Parameter(names = "--clear-instance-name", description = "delete any existing instance name without prompting")
 +    boolean clearInstanceName = false;
 +    @Parameter(names = "--instance-name", description = "the instance name, if not provided, will prompt")
 +    String cliInstanceName;
 +    @Parameter(names = "--password", description = "set the password on the command line")
 +    String cliPassword;
 +
 +    byte[] rootpass = null;
 +  }
 +
 +  public static void main(String[] args) {
 +    Opts opts = new Opts();
 +    opts.parseArgs(Initialize.class.getName(), args);
 +
 +    try {
 +      AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
 +      SecurityUtil.serverLogin(acuConf);
 +      Configuration conf = CachedConfiguration.getInstance();
 +
 +      VolumeManager fs = VolumeManagerImpl.get(acuConf);
 +
 +      if (opts.resetSecurity) {
 +        if (isInitialized(fs)) {
 +          opts.rootpass = getRootPassword(opts);
 +          initSecurity(opts, HdfsZooInstance.getInstance().getInstanceID());
 +        } else {
 +          log.fatal("Attempted to reset security on accumulo before it was initialized");
 +        }
 +      }
 +
 +      if (opts.addVolumes) {
 +        addVolumes(fs);
 +      }
 +
 +      if (!opts.resetSecurity && !opts.addVolumes)
 +        if (!doInit(opts, conf, fs))
 +          System.exit(-1);
 +    } catch (Exception e) {
 +      log.fatal(e, e);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +}


[20/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 103cfe8,0000000..547ccc7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@@ -1,1601 -1,0 +1,1603 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.nio.ByteBuffer;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.EnumSet;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +import java.util.Set;
 +import java.util.SortedSet;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.zip.ZipEntry;
 +import java.util.zip.ZipInputStream;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.NamespaceExistsException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableDeletedException;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.TableOfflineException;
 +import org.apache.accumulo.core.client.admin.DiskUsage;
 +import org.apache.accumulo.core.client.admin.FindMax;
 +import org.apache.accumulo.core.client.admin.TableOperations;
 +import org.apache.accumulo.core.client.admin.TimeType;
 +import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
 +import org.apache.accumulo.core.client.impl.thrift.TDiskUsage;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Constraint;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorUtil;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.master.thrift.FateOperation;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.metadata.MetadataServicer;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.accumulo.core.util.ByteBufferUtil;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.LocalityGroupUtil;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.core.util.NamingThreadFactory;
 +import org.apache.accumulo.core.util.OpTimer;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TApplicationException;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.transport.TTransportException;
 +
 +public class TableOperationsImpl extends TableOperationsHelper {
 +  private Instance instance;
 +  private Credentials credentials;
 +
 +  public static final String CLONE_EXCLUDE_PREFIX = "!";
 +
 +  private static final Logger log = Logger.getLogger(TableOperations.class);
 +
 +  /**
 +   * @param instance
 +   *          the connection information for this instance
 +   * @param credentials
 +   *          the username/password for this connection
 +   */
 +  public TableOperationsImpl(Instance instance, Credentials credentials) {
 +    ArgumentChecker.notNull(instance, credentials);
 +    this.instance = instance;
 +    this.credentials = credentials;
 +  }
 +
 +  /**
 +   * Retrieve a list of tables in Accumulo.
 +   *
 +   * @return List of tables in accumulo
 +   */
 +  @Override
 +  public SortedSet<String> list() {
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
 +    TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
 +    opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
 +    return tableNames;
 +  }
 +
 +  /**
 +   * A method to check if a table exists in Accumulo.
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @return true if the table exists
 +   */
 +  @Override
 +  public boolean exists(String tableName) {
 +    ArgumentChecker.notNull(tableName);
 +    if (tableName.equals(MetadataTable.NAME) || tableName.equals(RootTable.NAME))
 +      return true;
 +
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + " exists...");
 +    boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
 +    opTimer.stop("Checked existance of " + exists + " in %DURATION%");
 +    return exists;
 +  }
 +
 +  /**
 +   * Create a table with no special configuration
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableExistsException
 +   *           if the table already exists
 +   */
 +  @Override
 +  public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 +    create(tableName, true, TimeType.MILLIS);
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param limitVersion
 +   *          Enables/disables the versioning iterator, which will limit the number of Key versions kept.
 +   */
 +  @Override
 +  public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 +    create(tableName, limitVersion, TimeType.MILLIS);
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param timeType
 +   *          specifies logical or real-time based time recording for entries in the table
 +   * @param limitVersion
 +   *          Enables/disables the versioning iterator, which will limit the number of Key versions kept.
 +   */
 +  @Override
 +  public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 +    ArgumentChecker.notNull(tableName, timeType);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(UTF_8)));
 +
 +    Map<String,String> opts;
 +    if (limitVersion)
 +      opts = IteratorUtil.generateInitialTableProperties(limitVersion);
 +    else
 +      opts = Collections.emptyMap();
 +
 +    try {
 +      doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_CREATE, args, opts);
 +    } catch (TableNotFoundException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  private long beginFateOperation() throws ThriftSecurityException, TException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        return client.beginFateOperation(Tracer.traceInfo(), credentials.toThrift(instance));
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call beginFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  // This method is for retrying in the case of network failures; anything else it passes to the caller to deal with
 +  private void executeFateOperation(long opid, FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
 +      throws ThriftSecurityException, TException, ThriftTableOperationException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        client.executeFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid, op, args, opts, autoCleanUp);
 +        break;
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call executeFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  private String waitForFateOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        return client.waitForFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call waitForFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  private void finishFateOperation(long opid) throws ThriftSecurityException, TException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        client.finishFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
 +        break;
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call finishFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
 +      TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
 +    return doFateOperation(op, args, opts, true);
 +  }
 +
 +  String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
 +      TableExistsException, TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
 +    Long opid = null;
 +
 +    try {
 +      opid = beginFateOperation();
 +      executeFateOperation(opid, op, args, opts, !wait);
 +      if (!wait) {
 +        opid = null;
 +        return null;
 +      }
 +      String ret = waitForFateOperation(opid);
 +      return ret;
 +    } catch (ThriftSecurityException e) {
 +      String tableName = ByteBufferUtil.toString(args.get(0));
 +      switch (e.getCode()) {
 +        case TABLE_DOESNT_EXIST:
 +          throw new TableNotFoundException(null, tableName, "Target table does not exist");
 +        case NAMESPACE_DOESNT_EXIST:
 +          throw new NamespaceNotFoundException(null, tableName, "Target namespace does not exist");
 +        default:
 +          String tableInfo = Tables.getPrintableTableInfoFromName(instance, tableName);
 +          throw new AccumuloSecurityException(e.user, e.code, tableInfo, e);
 +      }
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case EXISTS:
 +          throw new TableExistsException(e);
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        case NAMESPACE_EXISTS:
 +          throw new NamespaceExistsException(e);
 +        case NAMESPACE_NOTFOUND:
 +          throw new NamespaceNotFoundException(e);
 +        case OFFLINE:
 +          throw new TableOfflineException(instance, null);
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (Exception e) {
 +      throw new AccumuloException(e.getMessage(), e);
 +    } finally {
 +      Tables.clearCache(instance);
 +      // always finish table op, even when exception
 +      if (opid != null)
 +        try {
 +          finishFateOperation(opid);
 +        } catch (Exception e) {
 +          log.warn(e.getMessage(), e);
 +        }
 +    }
 +  }
 +
 +  private static class SplitEnv {
 +    private String tableName;
 +    private String tableId;
 +    private ExecutorService executor;
 +    private CountDownLatch latch;
 +    private AtomicReference<Exception> exception;
 +
 +    SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
 +      this.tableName = tableName;
 +      this.tableId = tableId;
 +      this.executor = executor;
 +      this.latch = latch;
 +      this.exception = exception;
 +    }
 +  }
 +
 +  private class SplitTask implements Runnable {
 +
 +    private List<Text> splits;
 +    private SplitEnv env;
 +
 +    SplitTask(SplitEnv env, List<Text> splits) {
 +      this.env = env;
 +      this.splits = splits;
 +    }
 +
 +    @Override
 +    public void run() {
 +      try {
 +        if (env.exception.get() != null)
 +          return;
 +
 +        if (splits.size() <= 2) {
 +          addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
 +          for (int i = 0; i < splits.size(); i++)
 +            env.latch.countDown();
 +          return;
 +        }
 +
 +        int mid = splits.size() / 2;
 +
 +        // split the middle split point to ensure that child task split different tablets and can therefore
 +        // run in parallel
 +        addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
 +        env.latch.countDown();
 +
 +        env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
 +        env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
 +
 +      } catch (Exception e) {
 +        env.exception.compareAndSet(null, e);
 +      }
 +    }
 +
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param partitionKeys
 +   *          a sorted set of row key values to pre-split the table on
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 +    String tableId = Tables.getTableId(instance, tableName);
 +
 +    List<Text> splits = new ArrayList<Text>(partitionKeys);
 +    // should be sorted because we copied from a sorted set, but that makes assumptions about
 +    // how the copy was done so resort to be sure.
 +    Collections.sort(splits);
 +
 +    CountDownLatch latch = new CountDownLatch(splits.size());
 +    AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
 +
 +    ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
 +    try {
 +      executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
 +
 +      while (!latch.await(100, TimeUnit.MILLISECONDS)) {
 +        if (exception.get() != null) {
 +          executor.shutdownNow();
 +          Exception excep = exception.get();
 +          if (excep instanceof TableNotFoundException)
 +            throw (TableNotFoundException) excep;
 +          else if (excep instanceof AccumuloException)
 +            throw (AccumuloException) excep;
 +          else if (excep instanceof AccumuloSecurityException)
 +            throw (AccumuloSecurityException) excep;
 +          else if (excep instanceof RuntimeException)
 +            throw (RuntimeException) excep;
 +          else
 +            throw new RuntimeException(excep);
 +        }
 +      }
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    } finally {
 +      executor.shutdown();
 +    }
 +  }
 +
 +  private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
 +      TableNotFoundException, AccumuloServerException {
 +    TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(tableId));
 +
 +    for (Text split : partitionKeys) {
 +      boolean successful = false;
 +      int attempt = 0;
 +      long locationFailures = 0;
 +
 +      while (!successful) {
 +
 +        if (attempt > 0)
 +          UtilWaitThread.sleep(100);
 +
 +        attempt++;
 +
 +        TabletLocation tl = tabLocator.locateTablet(credentials, split, false, false);
 +
 +        if (tl == null) {
 +          if (!Tables.exists(instance, tableId))
 +            throw new TableNotFoundException(tableId, tableName, null);
 +          else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
 +            throw new TableOfflineException(instance, tableId);
 +          continue;
 +        }
 +
 +        try {
 +          TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, ServerConfigurationUtil.getConfiguration(instance));
 +          try {
 +            OpTimer opTimer = null;
 +            if (log.isTraceEnabled())
 +              opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
 +
 +            client.splitTablet(Tracer.traceInfo(), credentials.toThrift(instance), tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
 +
 +            // just split it, might as well invalidate it in the cache
 +            tabLocator.invalidateCache(tl.tablet_extent);
 +
 +            if (opTimer != null)
 +              opTimer.stop("Split tablet in %DURATION%");
 +          } finally {
 +            ThriftUtil.returnClient(client);
 +          }
 +
 +        } catch (TApplicationException tae) {
 +          throw new AccumuloServerException(tl.tablet_location, tae);
 +        } catch (TTransportException e) {
 +          tabLocator.invalidateCache(tl.tablet_location);
 +          continue;
 +        } catch (ThriftSecurityException e) {
 +          Tables.clearCache(instance);
 +          if (!Tables.exists(instance, tableId))
 +            throw new TableNotFoundException(tableId, tableName, null);
 +          throw new AccumuloSecurityException(e.user, e.code, e);
 +        } catch (NotServingTabletException e) {
 +          // Do not silently spin when we repeatedly fail to get the location for a tablet
 +          locationFailures++;
 +          if (5 == locationFailures || 0 == locationFailures % 50) {
 +            log.warn("Having difficulty locating hosting tabletserver for split " + split + " on table " + tableName + ". Seen " + locationFailures
 +                + " failures.");
 +          }
 +
 +          tabLocator.invalidateCache(tl.tablet_extent);
 +          continue;
 +        } catch (TException e) {
 +          tabLocator.invalidateCache(tl.tablet_location);
 +          continue;
 +        }
 +
 +        successful = true;
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    ArgumentChecker.notNull(tableName);
 +    ByteBuffer EMPTY = ByteBuffer.allocate(0);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
 +        end == null ? EMPTY : TextUtil.getByteBuffer(end));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_MERGE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  @Override
 +  public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    ArgumentChecker.notNull(tableName);
 +    ByteBuffer EMPTY = ByteBuffer.allocate(0);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
 +        end == null ? EMPTY : TextUtil.getByteBuffer(end));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE_RANGE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @return the split points (end-row names) for the table's current split profile
 +   */
 +  @Override
 +  public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
 +
 +    ArgumentChecker.notNull(tableName);
 +
 +    String tableId = Tables.getTableId(instance, tableName);
 +
 +    TreeMap<KeyExtent,String> tabletLocations = new TreeMap<KeyExtent,String>();
 +
 +    while (true) {
 +      try {
 +        tabletLocations.clear();
 +        // the following method throws AccumuloException for some conditions that should be retried
 +        MetadataServicer.forTableId(instance, credentials, tableId).getTabletLocations(tabletLocations);
 +        break;
 +      } catch (AccumuloSecurityException ase) {
 +        throw ase;
 +      } catch (Exception e) {
 +        if (!Tables.exists(instance, tableId)) {
 +          throw new TableNotFoundException(tableId, tableName, null);
 +        }
 +
 +        if (e instanceof RuntimeException && e.getCause() instanceof AccumuloSecurityException) {
 +          throw (AccumuloSecurityException) e.getCause();
 +        }
 +
 +        log.info(e.getMessage() + " ... retrying ...");
 +        UtilWaitThread.sleep(3000);
 +      }
 +    }
 +
 +    ArrayList<Text> endRows = new ArrayList<Text>(tabletLocations.size());
 +
 +    for (KeyExtent ke : tabletLocations.keySet())
 +      if (ke.getEndRow() != null)
 +        endRows.add(ke.getEndRow());
 +
 +    return endRows;
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
 +    try {
 +      return listSplits(tableName);
 +    } catch (AccumuloSecurityException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param maxSplits
 +   *          specifies the maximum number of splits to return
 +   * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
 +   */
 +  @Override
 +  public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
 +    Collection<Text> endRows = listSplits(tableName);
 +
 +    if (endRows.size() <= maxSplits)
 +      return endRows;
 +
 +    double r = (maxSplits + 1) / (double) (endRows.size());
 +    double pos = 0;
 +
 +    ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
 +
 +    int j = 0;
 +    for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
 +      pos += r;
 +      while (pos > 1) {
 +        subset.add(((ArrayList<Text>) endRows).get(i));
 +        j++;
 +        pos -= 1;
 +      }
 +    }
 +
 +    return subset;
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
 +    try {
 +      return listSplits(tableName, maxSplits);
 +    } catch (AccumuloSecurityException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Delete a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +  }
 +
 +  @Override
 +  public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
 +      throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
 +
 +    ArgumentChecker.notNull(srcTableName, newTableName);
 +
 +    String srcTableId = Tables.getTableId(instance, srcTableName);
 +
 +    if (flush)
 +      _flush(srcTableId, null, null, true);
 +
 +    if (propertiesToExclude == null)
 +      propertiesToExclude = Collections.emptySet();
 +
 +    if (propertiesToSet == null)
 +      propertiesToSet = Collections.emptyMap();
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    for (Entry<String,String> entry : propertiesToSet.entrySet()) {
 +      if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
 +        throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
 +      opts.put(entry.getKey(), entry.getValue());
 +    }
 +
 +    for (String prop : propertiesToExclude) {
 +      opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
 +    }
 +
 +    doTableFateOperation(newTableName, AccumuloException.class, FateOperation.TABLE_CLONE, args, opts);
 +  }
 +
 +  /**
 +   * Rename a table
 +   *
 +   * @param oldTableName
 +   *          the old table name
 +   * @param newTableName
 +   *          the new table name
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the old table name does not exist
 +   * @throws TableExistsException
 +   *           if the new table name already exists
 +   */
 +  @Override
 +  public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
 +      TableExistsException {
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    doTableFateOperation(oldTableName, TableNotFoundException.class, FateOperation.TABLE_RENAME, args, opts);
 +  }
 +
 +  /**
 +   * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
 +   */
 +  @Override
 +  @Deprecated
 +  public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
 +    try {
 +      flush(tableName, null, null, false);
 +    } catch (TableNotFoundException e) {
 +      throw new AccumuloException(e.getMessage(), e);
 +    }
 +  }
 +
 +  /**
 +   * Flush a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   */
 +  @Override
 +  public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +
 +    String tableId = Tables.getTableId(instance, tableName);
 +    _flush(tableId, start, end, wait);
 +  }
 +
 +  @Override
 +  public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
 +      AccumuloException {
 +    compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
 +  }
 +
 +  @Override
 +  public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
 +      TableNotFoundException, AccumuloException {
 +    ArgumentChecker.notNull(tableName);
 +    ByteBuffer EMPTY = ByteBuffer.allocate(0);
 +
 +    String tableId = Tables.getTableId(instance, tableName);
 +
 +    if (flush)
 +      _flush(tableId, start, end, true);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
 +        end == null ? EMPTY : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
 +
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doFateOperation(FateOperation.TABLE_COMPACT, args, opts, wait);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    } catch (NamespaceExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    } catch (NamespaceNotFoundException e) {
 +      throw new TableNotFoundException(null, tableName, "Namespace not found", e);
 +    }
 +  }
 +
 +  @Override
 +  public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
 +    String tableId = Tables.getTableId(instance, tableName);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
 +
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_CANCEL_COMPACT, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +  }
 +
 +  private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    try {
 +      long flushID;
 +
 +      // used to pass the table name. but the tableid associated with a table name could change between calls.
 +      // so pass the tableid to both calls
 +
 +      while (true) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnectionWithRetry(instance);
 +          flushID = client.initiateFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId);
 +          break;
 +        } catch (TTransportException tte) {
 +          log.debug("Failed to call initiateFlush, retrying ... ", tte);
 +          UtilWaitThread.sleep(100);
 +        } finally {
 +          MasterClient.close(client);
 +        }
 +      }
 +
 +      while (true) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnectionWithRetry(instance);
 +          client.waitForFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
 +              wait ? Long.MAX_VALUE : 1);
 +          break;
 +        } catch (TTransportException tte) {
 +          log.debug("Failed to call initiateFlush, retrying ... ", tte);
 +          UtilWaitThread.sleep(100);
 +        } finally {
 +          MasterClient.close(client);
 +        }
 +      }
 +    } catch (ThriftSecurityException e) {
 +      switch (e.getCode()) {
 +        case TABLE_DOESNT_EXIST:
 +          throw new TableNotFoundException(tableId, null, e.getMessage(), e);
 +        default:
 +          log.debug("flush security exception on table id " + tableId);
 +          throw new AccumuloSecurityException(e.user, e.code, e);
 +      }
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets a property on a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @param property
 +   *          the name of a per-table property
 +   * @param value
 +   *          the value to set a per-table property to
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   */
 +  @Override
 +  public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, property, value);
 +    try {
 +      MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
 +        @Override
 +        public void execute(MasterClientService.Client client) throws Exception {
 +          client.setTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property, value);
 +        }
 +      });
 +    } catch (TableNotFoundException e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  /**
 +   * Removes a property from a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @param property
 +   *          the name of a per-table property
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   */
 +  @Override
 +  public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, property);
 +    try {
 +      MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
 +        @Override
 +        public void execute(MasterClientService.Client client) throws Exception {
 +          client.removeTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property);
 +        }
 +      });
 +    } catch (TableNotFoundException e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  /**
 +   * Gets properties of a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @return all properties visible by this table (system and per-table properties)
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +    try {
 +      return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
 +        @Override
 +        public Map<String,String> execute(ClientService.Client client) throws Exception {
 +          return client.getTableConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), tableName);
 +        }
 +      }).entrySet();
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        case NAMESPACE_NOTFOUND:
 +          throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (AccumuloException e) {
 +      throw e;
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    }
 +
 +  }
 +
 +  /**
 +   * Sets a tables locality groups. A tables locality groups can be changed at any time.
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @param groups
 +   *          mapping of locality group names to column families in the locality group
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    // ensure locality groups do not overlap
 +    HashSet<Text> all = new HashSet<Text>();
 +    for (Entry<String,Set<Text>> entry : groups.entrySet()) {
 +
 +      if (!Collections.disjoint(all, entry.getValue())) {
 +        throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
 +      }
 +
 +      all.addAll(entry.getValue());
 +    }
 +
 +    for (Entry<String,Set<Text>> entry : groups.entrySet()) {
 +      Set<Text> colFams = entry.getValue();
 +      String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
 +      setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
 +    }
 +
 +    try {
 +      setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
 +    } catch (AccumuloException e) {
 +      if (e.getCause() instanceof TableNotFoundException)
 +        throw (TableNotFoundException) e.getCause();
 +      throw e;
 +    }
 +
 +    // remove anything extraneous
 +    String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
 +    for (Entry<String,String> entry : getProperties(tableName)) {
 +      String property = entry.getKey();
 +      if (property.startsWith(prefix)) {
 +        // this property configures a locality group, find out which
 +        // one:
 +        String[] parts = property.split("\\.");
 +        String group = parts[parts.length - 1];
 +
 +        if (!groups.containsKey(group)) {
 +          removeProperty(tableName, property);
 +        }
 +      }
 +    }
 +  }
 +
 +  /**
 +   *
 +   * Gets the locality groups currently set for a table.
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @return mapping of locality group names to column families in the locality group
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
 +    AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
 +    Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
 +
 +    Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
 +    for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
 +
 +      HashSet<Text> colFams = new HashSet<Text>();
 +
 +      for (ByteSequence bs : entry.getValue()) {
 +        colFams.add(new Text(bs.toArray()));
 +      }
 +
 +      groups2.put(entry.getKey(), colFams);
 +    }
 +
 +    return groups2;
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param range
 +   *          a range to split
 +   * @param maxSplits
 +   *          the maximum number of splits
 +   * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
 +      TableNotFoundException {
 +    ArgumentChecker.notNull(tableName, range);
 +    if (maxSplits < 1)
 +      throw new IllegalArgumentException("maximum splits must be >= 1");
 +    if (maxSplits == 1)
 +      return Collections.singleton(range);
 +
 +    Random random = new Random();
 +    Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
 +    String tableId = Tables.getTableId(instance, tableName);
 +    TabletLocator tl = TabletLocator.getLocator(instance, new Text(tableId));
 +    // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
 +    tl.invalidateCache();
 +    while (!tl.binRanges(credentials, Collections.singletonList(range), binnedRanges).isEmpty()) {
 +      if (!Tables.exists(instance, tableId))
 +        throw new TableDeletedException(tableId);
 +      if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
 +        throw new TableOfflineException(instance, tableId);
 +
 +      log.warn("Unable to locate bins for specified range. Retrying.");
 +      // sleep randomly between 100 and 200ms
 +      UtilWaitThread.sleep(100 + random.nextInt(100));
 +      binnedRanges.clear();
 +      tl.invalidateCache();
 +    }
 +
 +    // group key extents to get <= maxSplits
 +    LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
 +    List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
 +
 +    for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
 +      unmergedExtents.addAll(map.keySet());
 +
 +    // the sort method is efficient for linked list
 +    Collections.sort(unmergedExtents);
 +
 +    while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
 +      if (unmergedExtents.size() >= 2) {
 +        KeyExtent first = unmergedExtents.removeFirst();
 +        KeyExtent second = unmergedExtents.removeFirst();
 +        first.setEndRow(second.getEndRow());
 +        mergedExtents.add(first);
 +      } else {
 +        mergedExtents.addAll(unmergedExtents);
 +        unmergedExtents.clear();
 +        unmergedExtents.addAll(mergedExtents);
 +        mergedExtents.clear();
 +      }
 +
 +    }
 +
 +    mergedExtents.addAll(unmergedExtents);
 +
 +    Set<Range> ranges = new HashSet<Range>();
 +    for (KeyExtent k : mergedExtents)
 +      ranges.add(k.toDataRange().clip(range));
 +
 +    return ranges;
 +  }
 +
 +  // TODO Remove deprecation warning surppression when Hadoop1 support is dropped
 +  @SuppressWarnings("deprecation")
 +  private Path checkPath(String dir, String kind, String type) throws IOException, AccumuloException {
 +    Path ret;
 +    FileSystem fs = VolumeConfiguration.getVolume(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance)).getFileSystem();
 +
 +    if (dir.contains(":")) {
 +      ret = new Path(dir);
 +    } else {
 +      ret = fs.makeQualified(new Path(dir));
 +    }
 +
 +    if (!fs.exists(ret))
 +      throw new AccumuloException(kind + " import " + type + " directory " + dir + " does not exist!");
 +
 +    if (!fs.getFileStatus(ret).isDir()) {
 +      throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not a directory!");
 +    }
 +
 +    if (type.equals("failure")) {
 +      FileStatus[] listStatus = fs.listStatus(ret);
 +      if (listStatus != null && listStatus.length != 0) {
 +        throw new AccumuloException("Bulk import failure directory " + ret + " is not empty");
 +      }
 +    }
 +
 +    return ret;
 +  }
 +
 +  @Override
 +  public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
 +      TableNotFoundException, AccumuloException {
 +    ArgumentChecker.notNull(tableName, dir, failureDir);
 +    // check for table existance
 +    Tables.getTableId(instance, tableName);
 +
 +    Path dirPath = checkPath(dir, "Bulk", "");
 +    Path failPath = checkPath(failureDir, "Bulk", "failure");
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
-         ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(dirPath.toString().getBytes(UTF_8)),
++        ByteBuffer.wrap(failPath.toString().getBytes(UTF_8)), ByteBuffer.wrap((setTime + "").getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_BULK_IMPORT, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  private void waitForTableStateTransition(String tableId, TableState expectedState) throws AccumuloException, TableNotFoundException,
 +      AccumuloSecurityException {
 +
 +    Text startRow = null;
 +    Text lastRow = null;
 +
 +    while (true) {
 +
 +      if (Tables.getTableState(instance, tableId) != expectedState) {
 +        Tables.clearCache(instance);
 +        if (Tables.getTableState(instance, tableId) != expectedState) {
 +          if (!Tables.exists(instance, tableId))
 +            throw new TableDeletedException(tableId);
 +          throw new AccumuloException("Unexpected table state " + tableId + " " + Tables.getTableState(instance, tableId) + " != " + expectedState);
 +        }
 +      }
 +
 +      Range range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
 +      if (startRow == null || lastRow == null)
 +        range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
 +      else
 +        range = new Range(startRow, lastRow);
 +
 +      String metaTable = MetadataTable.NAME;
 +      if (tableId.equals(MetadataTable.ID))
 +        metaTable = RootTable.NAME;
 +      Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(metaTable, Authorizations.EMPTY);
 +      scanner = new IsolatedScanner(scanner);
 +      TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +      scanner.setRange(range);
 +
 +      RowIterator rowIter = new RowIterator(scanner);
 +
 +      KeyExtent lastExtent = null;
 +
 +      int total = 0;
 +      int waitFor = 0;
 +      int holes = 0;
 +      Text continueRow = null;
 +      MapCounter<String> serverCounts = new MapCounter<String>();
 +
 +      while (rowIter.hasNext()) {
 +        Iterator<Entry<Key,Value>> row = rowIter.next();
 +
 +        total++;
 +
 +        KeyExtent extent = null;
 +        String future = null;
 +        String current = null;
 +
 +        while (row.hasNext()) {
 +          Entry<Key,Value> entry = row.next();
 +          Key key = entry.getKey();
 +
 +          if (key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME))
 +            future = entry.getValue().toString();
 +
 +          if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME))
 +            current = entry.getValue().toString();
 +
 +          if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key))
 +            extent = new KeyExtent(key.getRow(), entry.getValue());
 +        }
 +
 +        if ((expectedState == TableState.ONLINE && current == null) || (expectedState == TableState.OFFLINE && (future != null || current != null))) {
 +          if (continueRow == null)
 +            continueRow = extent.getMetadataEntry();
 +          waitFor++;
 +          lastRow = extent.getMetadataEntry();
 +
 +          if (current != null)
 +            serverCounts.increment(current, 1);
 +          if (future != null)
 +            serverCounts.increment(future, 1);
 +        }
 +
 +        if (!extent.getTableId().toString().equals(tableId)) {
 +          throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
 +        }
 +
 +        if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
 +          holes++;
 +        }
 +
 +        lastExtent = extent;
 +      }
 +
 +      if (continueRow != null) {
 +        startRow = continueRow;
 +      }
 +
 +      if (holes > 0 || total == 0) {
 +        startRow = null;
 +        lastRow = null;
 +      }
 +
 +      if (waitFor > 0 || holes > 0 || total == 0) {
 +        long waitTime;
 +        long maxPerServer = 0;
 +        if (serverCounts.size() > 0) {
 +          maxPerServer = Collections.max(serverCounts.values());
 +          waitTime = maxPerServer * 10;
 +        } else
 +          waitTime = waitFor * 10;
 +        waitTime = Math.max(100, waitTime);
 +        waitTime = Math.min(5000, waitTime);
 +        log.trace("Waiting for " + waitFor + "(" + maxPerServer + ") tablets, startRow = " + startRow + " lastRow = " + lastRow + ", holes=" + holes
 +            + " sleeping:" + waitTime + "ms");
 +        UtilWaitThread.sleep(waitTime);
 +      } else {
 +        break;
 +      }
 +
 +    }
 +  }
 +
 +  @Override
 +  public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    offline(tableName, false);
 +  }
 +
 +  /**
 +   *
 +   * @param tableName
 +   *          the table to take offline
 +   * @throws AccumuloException
 +   *           when there is a general accumulo error
 +   * @throws AccumuloSecurityException
 +   *           when the user does not have the proper permissions
 +   */
 +  @Override
 +  public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +
 +    ArgumentChecker.notNull(tableName);
 +    String tableId = Tables.getTableId(instance, tableName);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_OFFLINE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +    if (wait)
 +      waitForTableStateTransition(tableId, TableState.OFFLINE);
 +  }
 +
 +  @Override
 +  public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    online(tableName, false);
 +  }
 +
 +  /**
 +   *
 +   * @param tableName
 +   *          the table to take online
 +   * @throws AccumuloException
 +   *           when there is a general accumulo error
 +   * @throws AccumuloSecurityException
 +   *           when the user does not have the proper permissions
 +   */
 +  @Override
 +  public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +    String tableId = Tables.getTableId(instance, tableName);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_ONLINE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +    if (wait)
 +      waitForTableStateTransition(tableId, TableState.ONLINE);
 +  }
 +
 +  /**
 +   * Clears the tablet locator cache for a specified table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws TableNotFoundException
 +   *           if table does not exist
 +   */
 +  @Override
 +  public void clearLocatorCache(String tableName) throws TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +    TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(Tables.getTableId(instance, tableName)));
 +    tabLocator.invalidateCache();
 +  }
 +
 +  /**
 +   * Get a mapping of table name to internal table id.
 +   *
 +   * @return the map from table name to internal table id
 +   */
 +  @Override
 +  public Map<String,String> tableIdMap() {
 +    return Tables.getNameToIdMap(instance);
 +  }
 +
 +  @Override
 +  public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
 +      throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, auths);
 +    Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, auths);
 +    return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
 +  }
 +
 +  @Override
 +  public List<DiskUsage> getDiskUsage(Set<String> tableNames) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    List<TDiskUsage> diskUsages = null;
 +    while (diskUsages == null) {
 +      Pair<String,Client> pair = null;
 +      try {
 +        // this operation may us a lot of memory... its likely that connections to tabletservers hosting metadata tablets will be cached, so do not use cached
 +        // connections
 +        pair = ServerClient.getConnection(instance, false);
 +        diskUsages = pair.getSecond().getDiskUsage(tableNames, credentials.toThrift(instance));
 +      } catch (ThriftTableOperationException e) {
 +        switch (e.getType()) {
 +          case NOTFOUND:
 +            throw new TableNotFoundException(e);
 +          case NAMESPACE_NOTFOUND:
 +            throw new TableNotFoundException(e.getTableName(), new NamespaceNotFoundException(e));
 +          default:
 +            throw new AccumuloException(e.description, e);
 +        }
 +      } catch (ThriftSecurityException e) {
 +        throw new AccumuloSecurityException(e.getUser(), e.getCode());
 +      } catch (TTransportException e) {
 +        // some sort of communication error occurred, retry
 +	if (pair == null) {
 +          log.debug("Disk usage request failed.  Pair is null.  Retrying request...", e);
 +	} else {
 +          log.debug("Disk usage request failed " + pair.getFirst() + ", retrying ... ", e);
 +	}
 +        UtilWaitThread.sleep(100);
 +      } catch (TException e) {
 +        // may be a TApplicationException which indicates error on the server side
 +        throw new AccumuloException(e);
 +      } finally {
 +        // must always return thrift connection
 +        if (pair != null)
 +          ServerClient.close(pair.getSecond());
 +      }
 +    }
 +
 +    List<DiskUsage> finalUsages = new ArrayList<DiskUsage>();
 +    for (TDiskUsage diskUsage : diskUsages) {
 +      finalUsages.add(new DiskUsage(new TreeSet<String>(diskUsage.getTables()), diskUsage.getUsage()));
 +    }
 +
 +    return finalUsages;
 +  }
 +
 +  public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
 +    HashMap<String,String> props = new HashMap<String,String>();
 +
 +    ZipInputStream zis = new ZipInputStream(fs.open(path));
 +    try {
 +      ZipEntry zipEntry;
 +      while ((zipEntry = zis.getNextEntry()) != null) {
 +        if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
-           BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++          BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
 +          try {
 +            String line;
 +            while ((line = in.readLine()) != null) {
 +              String sa[] = line.split("=", 2);
 +              props.put(sa[0], sa[1]);
 +            }
 +          } finally {
 +            in.close();
 +          }
 +
 +          break;
 +        }
 +      }
 +    } finally {
 +      zis.close();
 +    }
 +    return props;
 +  }
 +
 +  @Override
 +  public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, importDir);
 +
 +    try {
 +      importDir = checkPath(importDir, "Table", "").toString();
 +    } catch (IOException e) {
 +      throw new AccumuloException(e);
 +    }
 +
 +    try {
 +      FileSystem fs = new Path(importDir).getFileSystem(CachedConfiguration.getInstance());
 +      Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
 +
 +      for (Entry<String,String> entry : props.entrySet()) {
 +        if (Property.isClassProperty(entry.getKey()) && !entry.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
 +          Logger.getLogger(this.getClass()).info(
 +              "Imported table sets '" + entry.getKey() + "' to '" + entry.getValue() + "'.  Ensure this class is on Accumulo classpath.");
 +        }
 +      }
 +
 +    } catch (IOException ioe) {
 +      Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
 +    }
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(importDir.getBytes(UTF_8)));
 +
 +    Map<String,String> opts = Collections.emptyMap();
 +
 +    try {
 +      doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_IMPORT, args, opts);
 +    } catch (TableNotFoundException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +  }
 +
 +  @Override
 +  public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, exportDir);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
 +
 +    Map<String,String> opts = Collections.emptyMap();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_EXPORT, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  @Override
 +  public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
 +      AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, className, asTypeName);
 +
 +    try {
 +      return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
 +        @Override
 +        public Boolean execute(ClientService.Client client) throws Exception {
 +          return client.checkTableClass(Tracer.traceInfo(), credentials.toThrift(instance), tableName, className, asTypeName);
 +        }
 +      });
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        case NAMESPACE_NOTFOUND:
 +          throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (AccumuloException e) {
 +      throw e;
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
 +      TableNotFoundException {
 +    testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
 +    super.attachIterator(tableName, setting, scopes);
 +  }
 +
 +  @Override
 +  public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    testClassLoad(tableName, constraintClassName, Constraint.class.getName());
 +    return super.addConstraint(tableName, constraintClassName);
 +  }
 +
 +  private void doTableFateOperation(String tableName, Class<? extends Exception> namespaceNotFoundExceptionClass, FateOperation op, List<ByteBuffer> args,
 +      Map<String,String> opts) throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
 +    try {
 +      doFateOperation(op, args, opts);
 +    } catch (NamespaceExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    } catch (NamespaceNotFoundException e) {
 +      if (namespaceNotFoundExceptionClass == null) {
 +        // should not happen
 +        throw new AssertionError(e);
 +      } else if (AccumuloException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
 +        throw new AccumuloException("Cannot create table in non-existent namespace", e);
 +      } else if (TableNotFoundException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
 +        throw new TableNotFoundException(null, tableName, "Namespace not found", e);
 +      } else {
 +        // should not happen
 +        throw new AssertionError(e);
 +      }
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
index 32b6e28,6414193..4de95c3
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
@@@ -16,8 -16,9 +16,10 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.security.SecurityPermission;
 +import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.SortedMap;
@@@ -118,75 -41,36 +120,75 @@@ public class Tables 
      if (sm != null) {
        sm.checkPermission(TABLES_PERMISSION);
      }
 -    return ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
    }
 -  
 +
    private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
      ZooCache zc = getZooCache(instance);
 -    
 +
      List<String> tableIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES);
 -    
      TreeMap<String,String> tableMap = new TreeMap<String,String>();
 -    
 +    Map<String,String> namespaceIdToNameMap = new HashMap<String,String>();
 +
      for (String tableId : tableIds) {
 -      byte[] tblPath = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
 -      if (tblPath != null) {
 +      byte[] tableName = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
 +      byte[] nId = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
 +      String namespaceName = Namespaces.DEFAULT_NAMESPACE;
 +      // create fully qualified table name
 +      if (nId == null) {
 +        namespaceName = null;
 +      } else {
-         String namespaceId = new String(nId, Constants.UTF8);
++        String namespaceId = new String(nId, UTF_8);
 +        if (!namespaceId.equals(Namespaces.DEFAULT_NAMESPACE_ID)) {
 +          try {
 +            namespaceName = namespaceIdToNameMap.get(namespaceId);
 +            if (namespaceName == null) {
 +              namespaceName = Namespaces.getNamespaceName(instance, namespaceId);
 +              namespaceIdToNameMap.put(namespaceId, namespaceName);
 +            }
 +          } catch (NamespaceNotFoundException e) {
 +            log.error("Table (" + tableId + ") contains reference to namespace (" + namespaceId + ") that doesn't exist", e);
 +            continue;
 +          }
 +        }
 +      }
 +      if (tableName != null && namespaceName != null) {
-         String tableNameStr = qualified(new String(tableName, Constants.UTF8), namespaceName);
++        String tableNameStr = qualified(new String(tableName, UTF_8), namespaceName);
          if (nameAsKey)
 -          tableMap.put(new String(tblPath, UTF_8), tableId);
 +          tableMap.put(tableNameStr, tableId);
          else
 -          tableMap.put(tableId, new String(tblPath, UTF_8));
 +          tableMap.put(tableId, tableNameStr);
        }
      }
 -    
 +
      return tableMap;
    }
 -  
 +
    public static String getTableId(Instance instance, String tableName) throws TableNotFoundException {
 +    try {
 +      return _getTableId(instance, tableName);
 +    } catch (NamespaceNotFoundException e) {
 +      throw new TableNotFoundException(tableName, e);
 +    }
 +  }
 +
 +  public static String _getTableId(Instance instance, String tableName) throws NamespaceNotFoundException, TableNotFoundException {
      String tableId = getNameToIdMap(instance).get(tableName);
 -    if (tableId == null)
 -      throw new TableNotFoundException(tableId, tableName, null);
 +    if (tableId == null) {
 +      // maybe the table exist, but the cache was not updated yet... so try to clear the cache and check again
 +      clearCache(instance);
 +      tableId = getNameToIdMap(instance).get(tableName);
 +      if (tableId == null) {
 +        String namespace = qualify(tableName).getFirst();
 +        if (Namespaces.getNameToIdMap(instance).containsKey(namespace))
 +          throw new TableNotFoundException(null, tableName, null);
 +        else
 +          throw new NamespaceNotFoundException(null, namespace, null);
 +      }
 +    }
      return tableId;
    }
 -  
 +
    public static String getTableName(Instance instance, String tableId) throws TableNotFoundException {
      String tableName = getIdToNameMap(instance).get(tableId);
      if (tableName == null)
@@@ -250,65 -113,11 +252,65 @@@
      byte[] state = zc.get(statePath);
      if (state == null)
        return TableState.UNKNOWN;
 -    
 +
-     return TableState.valueOf(new String(state, Constants.UTF8));
+     return TableState.valueOf(new String(state, UTF_8));
    }
 -  
 +
    public static long getCacheResetCount() {
      return cacheResetCount.get();
    }
 +
 +  public static String qualified(String tableName) {
 +    return qualified(tableName, Namespaces.DEFAULT_NAMESPACE);
 +  }
 +
 +  public static String qualified(String tableName, String defaultNamespace) {
 +    Pair<String,String> qualifiedTableName = Tables.qualify(tableName, defaultNamespace);
 +    if (Namespaces.DEFAULT_NAMESPACE.equals(qualifiedTableName.getFirst()))
 +      return qualifiedTableName.getSecond();
 +    else
 +      return qualifiedTableName.toString("", ".", "");
 +  }
 +
 +  public static Pair<String,String> qualify(String tableName) {
 +    return qualify(tableName, Namespaces.DEFAULT_NAMESPACE);
 +  }
 +
 +  public static Pair<String,String> qualify(String tableName, String defaultNamespace) {
 +    if (!tableName.matches(Tables.VALID_NAME_REGEX))
 +      throw new IllegalArgumentException("Invalid table name '" + tableName + "'");
 +    if (MetadataTable.OLD_NAME.equals(tableName))
 +      tableName = MetadataTable.NAME;
 +    if (tableName.contains(".")) {
 +      String[] s = tableName.split("\\.", 2);
 +      return new Pair<String,String>(s[0], s[1]);
 +    }
 +    return new Pair<String,String>(defaultNamespace, tableName);
 +  }
 +
 +  /**
 +   * Returns the namespace id for a given table ID.
 +   * 
 +   * @param instance
 +   *          The Accumulo Instance
 +   * @param tableId
 +   *          The tableId
 +   * @return The namespace id which this table resides in.
 +   * @throws IllegalArgumentException
 +   *           if the table doesn't exist in ZooKeeper
 +   */
 +  public static String getNamespaceId(Instance instance, String tableId) throws IllegalArgumentException {
 +    ArgumentChecker.notNull(instance, tableId);
 +
 +    ZooCache zc = getZooCache(instance);
 +    byte[] n = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
 +
 +    // We might get null out of ZooCache if this tableID doesn't exist
 +    if (null == n) {
 +      throw new IllegalArgumentException("Table with id " + tableId + " does not exist");
 +    }
 +
-     return new String(n, Constants.UTF8);
++    return new String(n, UTF_8);
 +  }
 +
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index d58d1e4,a9f5b61..de7bcb2
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@@ -81,10 -82,10 +82,10 @@@ public class Writer 
        throw new IllegalArgumentException("Can not add empty mutations");
      
      while (true) {
 -      TabletLocation tabLoc = TabletLocator.getInstance(instance, table).locateTablet(new Text(m.getRow()), false, true, credentials);
 +      TabletLocation tabLoc = TabletLocator.getLocator(instance, table).locateTablet(credentials, new Text(m.getRow()), false, true);
        
        if (tabLoc == null) {
-         log.trace("No tablet location found for row " + new String(m.getRow(), Constants.UTF8));
+         log.trace("No tablet location found for row " + new String(m.getRow(), UTF_8));
          UtilWaitThread.sleep(500);
          continue;
        }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
index a071d07,0000000..bf365ef
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
@@@ -1,40 -1,0 +1,40 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.lexicoder;
 +
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
 +
 +/**
 + * This lexicoder encodes/decodes a given String to/from bytes without further processing. It can be combined with other encoders like the
 + * {@link ReverseLexicoder} to flip the default sort order.
 + * 
 + * @since 1.6.0
 + */
 +
 +public class StringLexicoder implements Lexicoder<String> {
 +  
 +  @Override
 +  public byte[] encode(String data) {
-     return data.getBytes(Constants.UTF8);
++    return data.getBytes(UTF_8);
 +  }
 +  
 +  @Override
 +  public String decode(byte[] data) {
-     return new String(data, Constants.UTF8);
++    return new String(data, UTF_8);
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 500047c,23540f2..563610a
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -27,8 -27,7 +29,7 @@@ import java.util.HashSet
  import java.util.List;
  import java.util.Set;
  
- import org.apache.accumulo.core.Constants;
 -import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
  import org.apache.accumulo.core.client.Instance;
  import org.apache.accumulo.core.client.IteratorSetting;
  import org.apache.accumulo.core.client.ZooKeeperInstance;
@@@ -203,24 -189,14 +204,24 @@@ public class RangeInputSplit extends In
      }
  
      if (in.readBoolean()) {
 -      String tokenClass = in.readUTF();
 -      byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
 -      byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
 -
 -      try {
 -        token = CredentialHelper.extractToken(tokenClass, tokenBytes);
 -      } catch (AccumuloSecurityException e) {
 -        throw new IOException(e);
 +      int ordinal = in.readInt();
 +      this.tokenSource = TokenSource.values()[ordinal];
 +
 +      switch (this.tokenSource) {
 +        case INLINE:
 +          String tokenClass = in.readUTF();
-           byte[] base64TokenBytes = in.readUTF().getBytes(Constants.UTF8);
++          byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
 +          byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
 +
 +          this.token = AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes);
 +          break;
 +
 +        case FILE:
 +          this.tokenFile = in.readUTF();
 +
 +          break;
 +        default:
 +          throw new IOException("Cannot parse unknown TokenSource ordinal");
        }
      }
  


[04/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 0e02f05,0000000..7158ea8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@@ -1,598 -1,0 +1,598 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.log;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_FINISH;
 +import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_START;
 +import static org.apache.accumulo.tserver.logger.LogEvents.DEFINE_TABLET;
 +import static org.apache.accumulo.tserver.logger.LogEvents.MANY_MUTATIONS;
 +import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
 +
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.EOFException;
 +import java.io.IOException;
 +import java.io.OutputStream;
 +import java.lang.reflect.InvocationTargetException;
 +import java.lang.reflect.Method;
 +import java.nio.channels.ClosedChannelException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.UUID;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.LinkedBlockingQueue;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.security.crypto.CryptoModule;
 +import org.apache.accumulo.core.security.crypto.CryptoModuleFactory;
 +import org.apache.accumulo.core.security.crypto.CryptoModuleParameters;
 +import org.apache.accumulo.core.security.crypto.DefaultCryptoModule;
 +import org.apache.accumulo.core.security.crypto.NoFlushOutputStream;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.tserver.TabletMutations;
 +import org.apache.accumulo.tserver.logger.LogFileKey;
 +import org.apache.accumulo.tserver.logger.LogFileValue;
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FSDataOutputStream;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * Wrap a connection to a logger.
 + *
 + */
 +public class DfsLogger {
 +  public static final String LOG_FILE_HEADER_V2 = "--- Log File Header (v2) ---";
 +  public static final String LOG_FILE_HEADER_V3 = "--- Log File Header (v3) ---";
 +
 +  private static Logger log = Logger.getLogger(DfsLogger.class);
 +
 +  public static class LogClosedException extends IOException {
 +    private static final long serialVersionUID = 1L;
 +
 +    public LogClosedException() {
 +      super("LogClosed");
 +    }
 +  }
 +
 +  /**
 +   * A well-timed tabletserver failure could result in an incomplete header written to a write-ahead log. This exception is thrown when the header cannot be
 +   * read from a WAL which should only happen when the tserver dies as described.
 +   */
 +  public static class LogHeaderIncompleteException extends IOException {
 +    private static final long serialVersionUID = 1l;
 +
 +    public LogHeaderIncompleteException(String msg) {
 +      super(msg);
 +    }
 +
 +    public LogHeaderIncompleteException(String msg, Throwable cause) {
 +      super(msg, cause);
 +    }
 +
 +    public LogHeaderIncompleteException(Throwable cause) {
 +      super(cause);
 +    }
 +  }
 +
 +  public static class DFSLoggerInputStreams {
 +
 +    private FSDataInputStream originalInput;
 +    private DataInputStream decryptingInputStream;
 +
 +    public DFSLoggerInputStreams(FSDataInputStream originalInput, DataInputStream decryptingInputStream) {
 +      this.originalInput = originalInput;
 +      this.decryptingInputStream = decryptingInputStream;
 +    }
 +
 +    public FSDataInputStream getOriginalInput() {
 +      return originalInput;
 +    }
 +
 +    public void setOriginalInput(FSDataInputStream originalInput) {
 +      this.originalInput = originalInput;
 +    }
 +
 +    public DataInputStream getDecryptingInputStream() {
 +      return decryptingInputStream;
 +    }
 +
 +    public void setDecryptingInputStream(DataInputStream decryptingInputStream) {
 +      this.decryptingInputStream = decryptingInputStream;
 +    }
 +  }
 +
 +  public interface ServerResources {
 +    AccumuloConfiguration getConfiguration();
 +
 +    VolumeManager getFileSystem();
 +
 +    Set<TServerInstance> getCurrentTServers();
 +  }
 +
 +  private final LinkedBlockingQueue<DfsLogger.LogWork> workQueue = new LinkedBlockingQueue<DfsLogger.LogWork>();
 +
 +  private final Object closeLock = new Object();
 +
 +  private static final DfsLogger.LogWork CLOSED_MARKER = new DfsLogger.LogWork(null);
 +
 +  private static final LogFileValue EMPTY = new LogFileValue();
 +
 +  private boolean closed = false;
 +
 +  private class LogSyncingTask implements Runnable {
 +
 +    @Override
 +    public void run() {
 +      ArrayList<DfsLogger.LogWork> work = new ArrayList<DfsLogger.LogWork>();
 +      boolean sawClosedMarker = false;
 +      while (!sawClosedMarker) {
 +        work.clear();
 +
 +        try {
 +          work.add(workQueue.take());
 +        } catch (InterruptedException ex) {
 +          continue;
 +        }
 +        workQueue.drainTo(work);
 +
 +        try {
 +          sync.invoke(logFile);
 +        } catch (Exception ex) {
 +          log.warn("Exception syncing " + ex);
 +          for (DfsLogger.LogWork logWork : work) {
 +            logWork.exception = ex;
 +          }
 +        }
 +
 +        for (DfsLogger.LogWork logWork : work)
 +          if (logWork == CLOSED_MARKER)
 +            sawClosedMarker = true;
 +          else
 +            logWork.latch.countDown();
 +      }
 +    }
 +  }
 +
 +  static class LogWork {
 +    CountDownLatch latch;
 +    volatile Exception exception;
 +
 +    public LogWork(CountDownLatch latch) {
 +      this.latch = latch;
 +    }
 +  }
 +
 +  public static class LoggerOperation {
 +    private final LogWork work;
 +
 +    public LoggerOperation(LogWork work) {
 +      this.work = work;
 +    }
 +
 +    public void await() throws IOException {
 +      try {
 +        work.latch.await();
 +      } catch (InterruptedException e) {
 +        throw new RuntimeException(e);
 +      }
 +
 +      if (work.exception != null) {
 +        if (work.exception instanceof IOException)
 +          throw (IOException) work.exception;
 +        else if (work.exception instanceof RuntimeException)
 +          throw (RuntimeException) work.exception;
 +        else
 +          throw new RuntimeException(work.exception);
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public boolean equals(Object obj) {
 +    // filename is unique
 +    if (obj == null)
 +      return false;
 +    if (obj instanceof DfsLogger)
 +      return getFileName().equals(((DfsLogger) obj).getFileName());
 +    return false;
 +  }
 +
 +  @Override
 +  public int hashCode() {
 +    // filename is unique
 +    return getFileName().hashCode();
 +  }
 +
 +  private final ServerResources conf;
 +  private FSDataOutputStream logFile;
 +  private DataOutputStream encryptingLogFile = null;
 +  private Method sync;
 +  private String logPath;
 +  private Daemon syncThread;
 +
 +  /* Track what's actually in +r/!0 for this logger ref */
 +  private String metaReference;
 +
 +  public DfsLogger(ServerResources conf) throws IOException {
 +    this.conf = conf;
 +  }
 +
 +  /**
 +   * Refernce a pre-existing log file.
 +   *
 +   * @param meta
 +   *          the cq for the "log" entry in +r/!0
 +   */
 +  public DfsLogger(ServerResources conf, String filename, String meta) throws IOException {
 +    this.conf = conf;
 +    this.logPath = filename;
 +    metaReference = meta;
 +  }
 +
 +  public static DFSLoggerInputStreams readHeaderAndReturnStream(VolumeManager fs, Path path, AccumuloConfiguration conf) throws IOException {
 +    FSDataInputStream input = fs.open(path);
 +    DataInputStream decryptingInput = null;
 +
 +    byte[] magic = DfsLogger.LOG_FILE_HEADER_V3.getBytes();
 +    byte[] magicBuffer = new byte[magic.length];
 +    try {
 +      input.readFully(magicBuffer);
 +      if (Arrays.equals(magicBuffer, magic)) {
 +        // additional parameters it needs from the underlying stream.
 +        String cryptoModuleClassname = input.readUTF();
 +        CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoModuleClassname);
 +
 +        // Create the parameters and set the input stream into those parameters
 +        CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
 +        params.setEncryptedInputStream(input);
 +
 +        // Create the plaintext input stream from the encrypted one
 +        params = cryptoModule.getDecryptingInputStream(params);
 +
 +        if (params.getPlaintextInputStream() instanceof DataInputStream) {
 +          decryptingInput = (DataInputStream) params.getPlaintextInputStream();
 +        } else {
 +          decryptingInput = new DataInputStream(params.getPlaintextInputStream());
 +        }
 +      } else {
 +        input.seek(0);
 +        byte[] magicV2 = DfsLogger.LOG_FILE_HEADER_V2.getBytes();
 +        byte[] magicBufferV2 = new byte[magicV2.length];
 +        input.readFully(magicBufferV2);
 +
 +        if (Arrays.equals(magicBufferV2, magicV2)) {
 +          // Log files from 1.5 dump their options in raw to the logger files. Since we don't know the class
 +          // that needs to read those files, we can make a couple of basic assumptions. Either it's going to be
 +          // the NullCryptoModule (no crypto) or the DefaultCryptoModule.
 +
 +          // If it's null, we won't have any parameters whatsoever. First, let's attempt to read
 +          // parameters
 +          Map<String,String> opts = new HashMap<String,String>();
 +          int count = input.readInt();
 +          for (int i = 0; i < count; i++) {
 +            String key = input.readUTF();
 +            String value = input.readUTF();
 +            opts.put(key, value);
 +          }
 +
 +          if (opts.size() == 0) {
 +            // NullCryptoModule, we're done
 +            decryptingInput = input;
 +          } else {
 +
 +            // The DefaultCryptoModule will want to read the parameters from the underlying file, so we will put the file back to that spot.
 +            org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory
 +                .getCryptoModule(DefaultCryptoModule.class.getName());
 +
 +            CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
 +
 +            input.seek(0);
 +            input.readFully(magicBufferV2);
 +            params.setEncryptedInputStream(input);
 +
 +            params = cryptoModule.getDecryptingInputStream(params);
 +            if (params.getPlaintextInputStream() instanceof DataInputStream) {
 +              decryptingInput = (DataInputStream) params.getPlaintextInputStream();
 +            } else {
 +              decryptingInput = new DataInputStream(params.getPlaintextInputStream());
 +            }
 +          }
 +
 +        } else {
 +
 +          input.seek(0);
 +          decryptingInput = input;
 +        }
 +
 +      }
 +    } catch (EOFException e) {
 +      log.warn("Got EOFException trying to read WAL header information, assuming the rest of the file (" + path + ") has no data.");
 +      // A TabletServer might have died before the (complete) header was written
 +      throw new LogHeaderIncompleteException(e);
 +    }
 +
 +    return new DFSLoggerInputStreams(input, decryptingInput);
 +  }
 +
 +  public synchronized void open(String address) throws IOException {
 +    String filename = UUID.randomUUID().toString();
 +    String logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
 +
 +    log.debug("DfsLogger.open() begin");
 +    VolumeManager fs = conf.getFileSystem();
 +
 +    logPath = fs.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
 +
 +    metaReference = toString();
 +    try {
 +      short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
 +      if (replication == 0)
 +        replication = fs.getDefaultReplication(new Path(logPath));
 +      long blockSize = conf.getConfiguration().getMemoryInBytes(Property.TSERV_WAL_BLOCKSIZE);
 +      if (blockSize == 0)
 +        blockSize = (long) (conf.getConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1);
 +      if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
 +        logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
 +      else
 +        logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
 +
 +      String syncMethod = conf.getConfiguration().get(Property.TSERV_WAL_SYNC_METHOD);
 +      try {
 +        // hsync: send data to datanodes and sync the data to disk
 +        sync = logFile.getClass().getMethod(syncMethod);
 +      } catch (Exception ex) {
 +        log.warn("Could not find configured " + syncMethod + " method, trying to fall back to old Hadoop sync method", ex);
 +
 +        try {
 +          // sync: send data to datanodes
 +          sync = logFile.getClass().getMethod("sync");
 +        } catch (Exception e) {
 +          throw new RuntimeException(e);
 +        }
 +      }
 +
 +      // Initialize the crypto operations.
 +      org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf
 +          .getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
 +
 +      // Initialize the log file with a header and the crypto params used to set up this log file.
-       logFile.write(LOG_FILE_HEADER_V3.getBytes(Constants.UTF8));
++      logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
 +
 +      CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
 +
 +      NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
 +      params.setPlaintextOutputStream(nfos);
 +
 +      // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
 +      // so that that crypto module can re-read its own parameters.
 +
 +      logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
 +
 +      params = cryptoModule.getEncryptingOutputStream(params);
 +      OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
 +
 +      // If the module just kicks back our original stream, then just use it, don't wrap it in
 +      // another data OutputStream.
 +      if (encipheringOutputStream == nfos) {
 +        log.debug("No enciphering, using raw output stream");
 +        encryptingLogFile = nfos;
 +      } else {
 +        log.debug("Enciphering found, wrapping in DataOutputStream");
 +        encryptingLogFile = new DataOutputStream(encipheringOutputStream);
 +      }
 +
 +      LogFileKey key = new LogFileKey();
 +      key.event = OPEN;
 +      key.tserverSession = filename;
 +      key.filename = filename;
 +      write(key, EMPTY);
 +      sync.invoke(logFile);
 +      log.debug("Got new write-ahead log: " + this);
 +    } catch (Exception ex) {
 +      if (logFile != null)
 +        logFile.close();
 +      logFile = null;
 +      encryptingLogFile = null;
 +      throw new IOException(ex);
 +    }
 +
 +    syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
 +    syncThread.setName("Accumulo WALog thread " + toString());
 +    syncThread.start();
 +  }
 +
 +  @Override
 +  public String toString() {
 +    String fileName = getFileName();
 +    if (fileName.contains(":"))
 +      return getLogger() + "/" + getFileName();
 +    return fileName;
 +  }
 +
 +  /**
 +   * get the cq needed to reference this logger's entry in +r/!0
 +   */
 +  public String getMeta() {
 +    if (null == metaReference) {
 +      throw new IllegalStateException("logger doesn't have meta reference. " + this);
 +    }
 +    return metaReference;
 +  }
 +
 +  public String getFileName() {
 +    return logPath.toString();
 +  }
 +
 +  public void close() throws IOException {
 +
 +    synchronized (closeLock) {
 +      if (closed)
 +        return;
 +      // after closed is set to true, nothing else should be added to the queue
 +      // CLOSED_MARKER should be the last thing on the queue, therefore when the
 +      // background thread sees the marker and exits there should be nothing else
 +      // to process... so nothing should be left waiting for the background
 +      // thread to do work
 +      closed = true;
 +      workQueue.add(CLOSED_MARKER);
 +    }
 +
 +    // wait for background thread to finish before closing log file
 +    if (syncThread != null) {
 +      try {
 +        syncThread.join();
 +      } catch (InterruptedException e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +
 +    // expect workq should be empty at this point
 +    if (workQueue.size() != 0) {
 +      log.error("WAL work queue not empty after sync thread exited");
 +      throw new IllegalStateException("WAL work queue not empty after sync thread exited");
 +    }
 +
 +    if (encryptingLogFile != null)
 +      try {
 +        logFile.close();
 +      } catch (IOException ex) {
 +        log.error(ex);
 +        throw new LogClosedException();
 +      }
 +  }
 +
 +  public synchronized void defineTablet(int seq, int tid, KeyExtent tablet) throws IOException {
 +    // write this log to the METADATA table
 +    final LogFileKey key = new LogFileKey();
 +    key.event = DEFINE_TABLET;
 +    key.seq = seq;
 +    key.tid = tid;
 +    key.tablet = tablet;
 +    try {
 +      write(key, EMPTY);
 +      sync.invoke(logFile);
 +    } catch (IllegalArgumentException e) {
 +      log.error("Signature of sync method changed. Accumulo is likely incompatible with this version of Hadoop.");
 +      throw new RuntimeException(e);
 +    } catch (IllegalAccessException e) {
 +      log.error("Could not invoke sync method due to permission error.");
 +      throw new RuntimeException(e);
 +    } catch (InvocationTargetException e) {
 +      Throwable cause = e.getCause();
 +      if (cause instanceof IOException) {
 +        throw (IOException) cause;
 +      } else if (cause instanceof RuntimeException) {
 +        throw (RuntimeException) cause;
 +      } else if (cause instanceof Error) {
 +        throw (Error) cause;
 +      } else {
 +        // Cause is null, or some other checked exception that was added later.
 +        throw new RuntimeException(e);
 +      }
 +    }
 +  }
 +
 +  private synchronized void write(LogFileKey key, LogFileValue value) throws IOException {
 +    key.write(encryptingLogFile);
 +    value.write(encryptingLogFile);
 +    encryptingLogFile.flush();
 +  }
 +
 +  public LoggerOperation log(int seq, int tid, Mutation mutation) throws IOException {
 +    return logManyTablets(Collections.singletonList(new TabletMutations(tid, seq, Collections.singletonList(mutation))));
 +  }
 +
 +  private LoggerOperation logFileData(List<Pair<LogFileKey,LogFileValue>> keys) throws IOException {
 +    DfsLogger.LogWork work = new DfsLogger.LogWork(new CountDownLatch(1));
 +    synchronized (DfsLogger.this) {
 +      try {
 +        for (Pair<LogFileKey,LogFileValue> pair : keys) {
 +          write(pair.getFirst(), pair.getSecond());
 +        }
 +      } catch (ClosedChannelException ex) {
 +        throw new LogClosedException();
 +      } catch (Exception e) {
 +        log.error(e, e);
 +        work.exception = e;
 +      }
 +    }
 +
 +    synchronized (closeLock) {
 +      // use a different lock for close check so that adding to work queue does not need
 +      // to wait on walog I/O operations
 +
 +      if (closed)
 +        throw new LogClosedException();
 +      workQueue.add(work);
 +    }
 +
 +    return new LoggerOperation(work);
 +  }
 +
 +  public LoggerOperation logManyTablets(List<TabletMutations> mutations) throws IOException {
 +    List<Pair<LogFileKey,LogFileValue>> data = new ArrayList<Pair<LogFileKey,LogFileValue>>();
 +    for (TabletMutations tabletMutations : mutations) {
 +      LogFileKey key = new LogFileKey();
 +      key.event = MANY_MUTATIONS;
 +      key.seq = tabletMutations.getSeq();
 +      key.tid = tabletMutations.getTid();
 +      LogFileValue value = new LogFileValue();
 +      value.mutations = tabletMutations.getMutations();
 +      data.add(new Pair<LogFileKey,LogFileValue>(key, value));
 +    }
 +    return logFileData(data);
 +  }
 +
 +  public LoggerOperation minorCompactionFinished(int seq, int tid, String fqfn) throws IOException {
 +    LogFileKey key = new LogFileKey();
 +    key.event = COMPACTION_FINISH;
 +    key.seq = seq;
 +    key.tid = tid;
 +    return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
 +  }
 +
 +  public LoggerOperation minorCompactionStarted(int seq, int tid, String fqfn) throws IOException {
 +    LogFileKey key = new LogFileKey();
 +    key.event = COMPACTION_START;
 +    key.seq = seq;
 +    key.tid = tid;
 +    key.filename = fqfn;
 +    return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
 +  }
 +
 +  public String getLogger() {
 +    String parts[] = logPath.split("/");
 +    return StringUtil.join(Arrays.asList(parts[parts.length - 2].split("[+]")), ":");
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
index 997f71b,0000000..2c807fd
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
@@@ -1,96 -1,0 +1,97 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.logger;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.DataInput;
 +import java.io.DataOutput;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.server.data.ServerMutation;
 +import org.apache.hadoop.io.Writable;
 +
 +public class LogFileValue implements Writable {
 +  
 +  private static final List<Mutation> empty = Collections.emptyList();
 +  
 +  public List<Mutation> mutations = empty;
 +  
 +  @Override
 +  public void readFields(DataInput in) throws IOException {
 +    int count = in.readInt();
 +    mutations = new ArrayList<Mutation>(count);
 +    for (int i = 0; i < count; i++) {
 +      ServerMutation mutation = new ServerMutation();
 +      mutation.readFields(in);
 +      mutations.add(mutation);
 +    }
 +  }
 +  
 +  @Override
 +  public void write(DataOutput out) throws IOException {
 +    out.writeInt(mutations.size());
 +    for (Mutation m : mutations) {
 +      m.write(out);
 +    }
 +  }
 +  
 +  public static void print(LogFileValue value) {
 +    System.out.println(value.toString());
 +  }
 +  
 +  private static String displayLabels(byte[] labels) {
-     String s = new String(labels, Constants.UTF8);
++    String s = new String(labels, UTF_8);
 +    s = s.replace("&", " & ");
 +    s = s.replace("|", " | ");
 +    return s;
 +  }
 +  
 +  public static String format(LogFileValue lfv, int maxMutations) {
 +    if (lfv.mutations.size() == 0)
 +      return "";
 +    StringBuilder builder = new StringBuilder();
 +    builder.append(lfv.mutations.size() + " mutations:\n");
 +    int i = 0;
 +    for (Mutation m : lfv.mutations) {
 +      if (i++ >= maxMutations) {
 +        builder.append("...");
 +        break;
 +      }
-       builder.append("  ").append(new String(m.getRow(), Constants.UTF8)).append("\n");
++      builder.append("  ").append(new String(m.getRow(), UTF_8)).append("\n");
 +      for (ColumnUpdate update : m.getUpdates()) {
 +        String value = new String(update.getValue());
-         builder.append("      ").append(new String(update.getColumnFamily(), Constants.UTF8)).append(":")
-                 .append(new String(update.getColumnQualifier(), Constants.UTF8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
++        builder.append("      ").append(new String(update.getColumnFamily(), UTF_8)).append(":")
++                .append(new String(update.getColumnQualifier(), UTF_8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
 +                .append(update.getTimestamp()).append(" [").append(displayLabels(update.getColumnVisibility())).append("] ")
 +                .append(update.isDeleted() ? "<deleted>" : value).append("\n");
 +      }
 +    }
 +    return builder.toString();
 +  }
 +  
 +  @Override
 +  public String toString() {
 +    return format(this, 5);
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index c3f4fd0,0000000..29eefc8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -1,178 -1,0 +1,179 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.logger;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.DataInputStream;
 +import java.io.EOFException;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Set;
 +import java.util.regex.Matcher;
 +import java.util.regex.Pattern;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
 +import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException;
 +import org.apache.accumulo.tserver.log.MultiReader;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.JCommander;
 +import com.beust.jcommander.Parameter;
 +
 +public class LogReader {
 +  private static final Logger log = Logger.getLogger(LogReader.class);
 +
 +  static class Opts extends Help {
 +    @Parameter(names = "-r", description = "print only mutations associated with the given row")
 +    String row;
 +    @Parameter(names = "-m", description = "limit the number of mutations printed per row")
 +    int maxMutations = 5;
 +    @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
 +    String extent;
 +    @Parameter(names = "-p", description = "search for a row that matches the given regex")
 +    String regexp;
 +    @Parameter(description = "<logfile> { <logfile> ... }")
 +    List<String> files = new ArrayList<String>();
 +  }
 +
 +  /**
 +   * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
 +   *
 +   * @param args
 +   *          - first argument is the file to print
 +   */
 +  public static void main(String[] args) throws IOException {
 +    Opts opts = new Opts();
 +    opts.parseArgs(LogReader.class.getName(), args);
 +    VolumeManager fs = VolumeManagerImpl.get();
 +
 +    Matcher rowMatcher = null;
 +    KeyExtent ke = null;
 +    Text row = null;
 +    if (opts.files.isEmpty()) {
 +      new JCommander(opts).usage();
 +      return;
 +    }
 +    if (opts.row != null)
 +      row = new Text(opts.row);
 +    if (opts.extent != null) {
 +      String sa[] = opts.extent.split(";");
 +      ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
 +    }
 +    if (opts.regexp != null) {
 +      Pattern pattern = Pattern.compile(opts.regexp);
 +      rowMatcher = pattern.matcher("");
 +    }
 +
 +    Set<Integer> tabletIds = new HashSet<Integer>();
 +
 +    for (String file : opts.files) {
 +
 +      Path path = new Path(file);
 +      LogFileKey key = new LogFileKey();
 +      LogFileValue value = new LogFileValue();
 +
 +      if (fs.isFile(path)) {
 +        // read log entries from a simple hdfs file
 +        DFSLoggerInputStreams streams;
 +        try {
 +          streams = DfsLogger.readHeaderAndReturnStream(fs, path, ServerConfiguration.getSiteConfiguration());
 +        } catch (LogHeaderIncompleteException e) {
 +          log.warn("Could not read header for " + path + ". Ignoring...");
 +          continue;
 +        }
 +        DataInputStream input = streams.getDecryptingInputStream();
 +
 +        try {
 +          while (true) {
 +            try {
 +              key.readFields(input);
 +              value.readFields(input);
 +            } catch (EOFException ex) {
 +              break;
 +            }
 +            printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
 +          }
 +        } finally {
 +          input.close();
 +        }
 +      } else {
 +        // read the log entries sorted in a map file
 +        MultiReader input = new MultiReader(fs, path);
 +        while (input.next(key, value)) {
 +          printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
 +        }
 +      }
 +    }
 +  }
 +
 +  public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
 +
 +    if (ke != null) {
 +      if (key.event == LogEvents.DEFINE_TABLET) {
 +        if (key.tablet.equals(ke)) {
 +          tabletIds.add(key.tid);
 +        } else {
 +          return;
 +        }
 +      } else if (!tabletIds.contains(key.tid)) {
 +        return;
 +      }
 +    }
 +
 +    if (row != null || rowMatcher != null) {
 +      if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
 +        boolean found = false;
 +        for (Mutation m : value.mutations) {
 +          if (row != null && new Text(m.getRow()).equals(row)) {
 +            found = true;
 +            break;
 +          }
 +
 +          if (rowMatcher != null) {
-             rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
++            rowMatcher.reset(new String(m.getRow(), UTF_8));
 +            if (rowMatcher.matches()) {
 +              found = true;
 +              break;
 +            }
 +          }
 +        }
 +
 +        if (!found)
 +          return;
 +      } else {
 +        return;
 +      }
 +
 +    }
 +
 +    System.out.println(key);
 +    System.out.println(LogFileValue.format(value, maxMutations));
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
index a679c48,ef6311b..3c20858
--- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
@@@ -71,98 -72,71 +72,98 @@@ public class TestBinaryRows 
      return l;
    }
    
 -  static class Opts extends ClientOnRequiredTable {
 +  public static class Opts extends ClientOnRequiredTable {
      @Parameter(names="--mode", description="either 'ingest', 'delete', 'randomLookups', 'split', 'verify', 'verifyDeleted'", required=true)
 -    String mode;
 +    public String mode;
      @Parameter(names="--start", description="the lowest numbered row")
 -    long start = 0;
 +    public long start = 0;
      @Parameter(names="--count", description="number of rows to ingest", required=true)
 -    long num = 0;
 +    public long num = 0;
    }
    
 -  public static void main(String[] args) {
 -    Opts opts = new Opts();
 -    BatchWriterOpts bwOpts = new BatchWriterOpts();
 -    ScannerOpts scanOpts = new ScannerOpts();
 -    opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
 +  public static void runTest(Connector connector, Opts opts, BatchWriterOpts bwOpts, ScannerOpts scanOpts) throws Exception {
      
 -    try {
 -      Connector connector = opts.getConnector();
 -      
 -      final Text CF = new Text("cf"), CQ = new Text("cq");
 -      final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
 +    final Text CF = new Text("cf"), CQ = new Text("cq");
-     final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
++    final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
 +    if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
 +      BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
 +      boolean delete = opts.mode.equals("delete");
        
 -      if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
 -        BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
 -        boolean delete = opts.mode.equals("delete");
 -        
 -        for (long i = 0; i < opts.num; i++) {
 -          byte[] row = encodeLong(i + opts.start);
 -          String value = "" + (i + opts.start);
 -          
 -          Mutation m = new Mutation(new Text(row));
 -          if (delete) {
 -            m.putDelete(CF, CQ);
 -          } else {
 -            m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
 -          }
 -          bw.addMutation(m);
 +      for (long i = 0; i < opts.num; i++) {
 +        byte[] row = encodeLong(i + opts.start);
 +        String value = "" + (i + opts.start);
 +        
 +        Mutation m = new Mutation(new Text(row));
 +        if (delete) {
 +          m.putDelete(CF, CQ);
 +        } else {
-           m.put(CF, CQ, new Value(value.getBytes(Constants.UTF8)));
++          m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
          }
 -        
 -        bw.close();
 -      } else if (opts.mode.equals("verifyDeleted")) {
 -        Scanner s = connector.createScanner(opts.tableName, opts.auths);
 -        s.setBatchSize(scanOpts.scanBatchSize);
 -        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 -        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
 -        s.setBatchSize(50000);
 -        s.setRange(new Range(startKey, stopKey));
 -        
 -        for (Entry<Key,Value> entry : s) {
 -          System.err.println("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
 -          System.err.println("exiting...");
 -          System.exit(1);
 -        }
 -        
 -      } else if (opts.mode.equals("verify")) {
 -        long t1 = System.currentTimeMillis();
 +        bw.addMutation(m);
 +      }
 +      
 +      bw.close();
 +    } else if (opts.mode.equals("verifyDeleted")) {
 +      Scanner s = connector.createScanner(opts.tableName, opts.auths);
 +      s.setBatchSize(scanOpts.scanBatchSize);
 +      Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 +      Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
 +      s.setBatchSize(50000);
 +      s.setRange(new Range(startKey, stopKey));
 +      
 +      for (Entry<Key,Value> entry : s) {
 +        throw new Exception("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
 +      }
 +      
 +    } else if (opts.mode.equals("verify")) {
 +      long t1 = System.currentTimeMillis();
 +      
 +      Scanner s = connector.createScanner(opts.tableName, opts.auths);
 +      Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 +      Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
 +      s.setBatchSize(scanOpts.scanBatchSize);
 +      s.setRange(new Range(startKey, stopKey));
 +      
 +      long i = opts.start;
 +      
 +      for (Entry<Key,Value> e : s) {
 +        Key k = e.getKey();
 +        Value v = e.getValue();
 +
 +        checkKeyValue(i, k, v);
 +
 +        i++;
 +      }
 +      
 +      if (i != opts.start + opts.num) {
 +        throw new Exception("ERROR : did not see expected number of rows, saw " + (i - opts.start) + " expected " + opts.num);
 +      }
 +      
 +      long t2 = System.currentTimeMillis();
 +      
 +      System.out.printf("time : %9.2f secs%n", ((t2 - t1) / 1000.0));
 +      System.out.printf("rate : %9.2f entries/sec%n", opts.num / ((t2 - t1) / 1000.0));
 +      
 +    } else if (opts.mode.equals("randomLookups")) {
 +      int numLookups = 1000;
 +      
 +      Random r = new Random();
 +      
 +      long t1 = System.currentTimeMillis();
 +      
 +      for (int i = 0; i < numLookups; i++) {
 +        long row = ((r.nextLong() & 0x7fffffffffffffffl) % opts.num) + opts.start;
          
          Scanner s = connector.createScanner(opts.tableName, opts.auths);
 -        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 -        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
          s.setBatchSize(scanOpts.scanBatchSize);
 +        Key startKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
 +        Key stopKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], 0);
          s.setRange(new Range(startKey, stopKey));
          
 -        long i = opts.start;
 +        Iterator<Entry<Key,Value>> si = s.iterator();
          
 -        for (Entry<Key,Value> e : s) {
 +        if (si.hasNext()) {
 +          Entry<Key,Value> e = si.next();
            Key k = e.getKey();
            Value v = e.getValue();
            

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestIngest.java
index bc3ff05,5d8e268..c3e77af
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@@ -16,16 -16,14 +16,17 @@@
   */
  package org.apache.accumulo.test;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 +import java.io.IOException;
  import java.util.Map.Entry;
  import java.util.Random;
  import java.util.Set;
  import java.util.TreeSet;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.BatchWriterOpts;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
  import org.apache.accumulo.core.client.BatchWriter;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index 7a848a3,125ef5f..16f0b3f
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@@ -16,11 -16,11 +16,12 @@@
   */
  package org.apache.accumulo.test;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.util.ArrayList;
 +import java.util.List;
  import java.util.Map.Entry;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.BatchWriterOpts;
  import org.apache.accumulo.core.cli.ScannerOpts;
  import org.apache.accumulo.core.client.AccumuloException;
@@@ -101,8 -97,8 +102,8 @@@ public class TestMultiTableIngest 
        
        // populate
        for (int i = 0; i < opts.count; i++) {
 -        Mutation m = new Mutation(new Text(String.format("%05d", i)));
 +        Mutation m = new Mutation(new Text(String.format("%06d", i)));
-         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
+         m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
          b.getBatchWriter(tableNames.get(i % tableNames.size())).addMutation(m);
        }
        try {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
index 6082ebe,5962220..4f88c1b
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
@@@ -26,7 -28,7 +28,6 @@@ import java.util.Map.Entry
  import java.util.Random;
  import java.util.zip.CRC32;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.Connector;
  import org.apache.accumulo.core.client.Scanner;
  import org.apache.accumulo.core.data.Key;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
index 4bd07ca,6124a3d..11d19c5
--- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
@@@ -82,9 -82,9 +83,9 @@@ public class UndefinedAnalyzer 
          parseLog(log);
        }
      }
 -    
 +
      private void parseLog(File log) throws Exception {
-       BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), Constants.UTF8));
+       BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), UTF_8));
        String line;
        TreeMap<Long,Long> tm = null;
        try {
@@@ -173,10 -172,10 +174,10 @@@
        SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM");
        String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + "";
        String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + "";
 -      
 +
        for (File masterLog : masterLogs) {
 -        
 +
-         BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), Constants.UTF8));
+         BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), UTF_8));
          String line;
          try {
            while ((line = reader.readLine()) != null) {
@@@ -256,11 -252,11 +257,11 @@@
    public static void main(String[] args) throws Exception {
      Opts opts = new Opts();
      BatchScannerOpts bsOpts = new BatchScannerOpts();
 -    opts.parseArgs(UndefinedAnalyzer.class.getName(), args, opts);
 -    
 +    opts.parseArgs(UndefinedAnalyzer.class.getName(), args, bsOpts);
 +
      List<UndefinedNode> undefs = new ArrayList<UndefinedNode>();
 -    
 +
-     BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, Constants.UTF8));
+     BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, UTF_8));
      String line;
      while ((line = reader.readLine()) != null) {
        String[] tokens = line.split("\\s");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index f26c8d7,166e2ad..949fc52
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -16,6 -16,10 +16,8 @@@
   */
  package org.apache.accumulo.test.functional;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
 -import java.net.InetAddress;
 -import java.net.InetSocketAddress;
  import java.util.HashMap;
  import java.util.Random;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index cb17340,33ef0b5..72efee3
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -24,7 -26,7 +26,6 @@@ import java.util.Random
  import java.util.TreeSet;
  import java.util.UUID;
  
--import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.BatchScanner;
  import org.apache.accumulo.core.client.BatchWriter;
  import org.apache.accumulo.core.client.BatchWriterConfig;
@@@ -104,10 -100,10 +105,10 @@@ public class MetadataBatchScanTest 
          
          String dir = "/t-" + UUID.randomUUID();
          
-         TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
 -        Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
++        TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
          
          for (int i = 0; i < 5; i++) {
-           mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
 -          mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
++          mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
          }
          
          bw.addMutation(mut);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
index 5756934,0191b50..e4ce3f3
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
@@@ -41,9 -37,7 +43,8 @@@ import javax.xml.parsers.DocumentBuilde
  import javax.xml.validation.Schema;
  import javax.xml.validation.SchemaFactory;
  
- import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.util.SimpleThreadPool;
  import org.apache.log4j.Level;
  import org.w3c.dom.Document;
  import org.w3c.dom.Element;
@@@ -201,163 -193,109 +202,163 @@@ public class Module extends Node 
      if (fixture != null) {
        fixture.setUp(state);
      }
 -    
 -    Node initNode = getNode(initNodeId);
 -    
 -    boolean test = false;
 -    if (initNode instanceof Test) {
 -      startTimer(initNode);
 -      test = true;
 -    }
 -    initNode.visit(state, getProps(initNodeId));
 -    if (test)
 -      stopTimer(initNode);
 -    
 -    state.visitedNode();
 -    // update aliases
 -    Set<String> aliases;
 -    if ((aliases = aliasMap.get(initNodeId)) != null)
 -      for (String alias : aliases) {
 -        ((Alias) nodes.get(alias)).update(initNodeId);
 -      }
 -    
 -    String curNodeId = initNodeId;
 -    int numHops = 0;
 -    long startTime = System.currentTimeMillis() / 1000;
 -    while (true) {
 -      // check if END state was reached
 -      if (curNodeId.equalsIgnoreCase("END")) {
 -        log.debug("reached END state");
 -        break;
 -      }
 -      // check if maxSec was reached
 -      long curTime = System.currentTimeMillis() / 1000;
 -      if ((curTime - startTime) > maxSec) {
 -        log.debug("reached maxSec(" + maxSec + ")");
 -        break;
 -      }
 -      // check if maxHops was reached
 -      if (numHops > maxHops) {
 -        log.debug("reached maxHops(" + maxHops + ")");
 -        break;
 -      }
 -      numHops++;
 -      
 -      if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
 -        throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
 -      }
 -      AdjList adj = adjMap.get(curNodeId);
 -      String nextNodeId = adj.randomNeighbor();
 -      Node nextNode = getNode(nextNodeId);
 -      if (nextNode instanceof Alias) {
 -        nextNodeId = ((Alias) nextNode).getTargetId();
 -        nextNode = ((Alias) nextNode).get();
 +
 +    ExecutorService service = new SimpleThreadPool(1, "RandomWalk Runner");
 +
 +    try {
 +      Node initNode = getNode(initNodeId);
 +
 +      boolean test = false;
 +      if (initNode instanceof Test) {
 +        startTimer(initNode);
 +        test = true;
        }
 -      Properties nodeProps = getProps(nextNodeId);
 -      try {
 -        test = false;
 -        if (nextNode instanceof Test) {
 -          startTimer(nextNode);
 -          test = true;
 +      initNode.visit(state, getProps(initNodeId));
 +      if (test)
 +        stopTimer(initNode);
 +
 +      state.visitedNode();
 +      // update aliases
 +      Set<String> aliases;
 +      if ((aliases = aliasMap.get(initNodeId)) != null)
 +        for (String alias : aliases) {
 +          ((Alias) nodes.get(alias)).update(initNodeId);
          }
 -        nextNode.visit(state, nodeProps);
 -        if (test)
 -          stopTimer(nextNode);
 -      } catch (Exception e) {
 -        log.debug("Connector belongs to user: " + state.getConnector().whoami());
 -        log.debug("Exception occured at: " + System.currentTimeMillis());
 -        log.debug("Properties for node: " + nextNodeId);
 -        for (Entry<Object,Object> entry : nodeProps.entrySet()) {
 -          log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +
 +      String curNodeId = initNodeId;
 +      int numHops = 0;
 +      long startTime = System.currentTimeMillis() / 1000;
 +      while (true) {
 +        // check if END state was reached
 +        if (curNodeId.equalsIgnoreCase("END")) {
 +          log.debug("reached END state");
 +          break;
          }
 -        log.debug("Overall Properties");
 -        for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
 -          log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +        // check if maxSec was reached
 +        long curTime = System.currentTimeMillis() / 1000;
 +        if ((curTime - startTime) > maxSec) {
 +          log.debug("reached maxSec(" + maxSec + ")");
 +          break;
          }
 -        log.debug("State information");
 -        for (String key : new TreeSet<String>(state.getMap().keySet()))  {
 -          Object value = state.getMap().get(key);
 -          String logMsg = "  " + key + ": ";
 -          if (value == null)
 -            logMsg += "null";
 -          else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
 -            logMsg += value;
 -          else if (value instanceof byte[])
 -            logMsg += new String((byte[])value, UTF_8);
 -          else if (value instanceof PasswordToken)
 -            logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
 -          else
 -            logMsg += value.getClass()+ " - " + value;
 -          
 -          log.debug(logMsg);
 +
 +        // The number of seconds before the test should exit
 +        long secondsRemaining = maxSec - (curTime - startTime);
 +
 +        // check if maxHops was reached
 +        if (numHops > maxHops) {
 +          log.debug("reached maxHops(" + maxHops + ")");
 +          break;
          }
 -        throw new Exception("Error running node " + nextNodeId, e);
 -      }
 -      state.visitedNode();
 -      
 -      // update aliases
 -      if ((aliases = aliasMap.get(curNodeId)) != null)
 -        for (String alias : aliases) {
 -          ((Alias) nodes.get(alias)).update(curNodeId);
 +        numHops++;
 +
 +        if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
 +          throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
 +        }
 +        AdjList adj = adjMap.get(curNodeId);
 +        String nextNodeId = adj.randomNeighbor();
 +        final Node nextNode;
 +        Node nextNodeOrAlias = getNode(nextNodeId);
 +        if (nextNodeOrAlias instanceof Alias) {
 +          nextNodeId = ((Alias) nextNodeOrAlias).getTargetId();
 +          nextNode = ((Alias) nextNodeOrAlias).get();
 +        } else {
 +          nextNode = nextNodeOrAlias;
 +        }
 +        final Properties nodeProps = getProps(nextNodeId);
 +        try {
 +          test = false;
 +          if (nextNode instanceof Test) {
 +            startTimer(nextNode);
 +            test = true;
 +          }
 +
 +          // Wrap the visit of the next node in the module in a callable that returns a thrown exception
 +          FutureTask<Exception> task = new FutureTask<Exception>(new Callable<Exception>() {
 +
 +            @Override
 +            public Exception call() throws Exception {
 +              try {
 +                nextNode.visit(state, nodeProps);
 +                return null;
 +              } catch (Exception e) {
 +                return e;
 +              }
 +            }
 +
 +          });
 +
 +          // Run the task (should execute immediately)
 +          service.submit(task);
 +
 +          Exception nodeException;
 +          try {
 +            // Bound the time we'll wait for the node to complete
 +            nodeException = task.get(secondsRemaining, TimeUnit.SECONDS);
 +          } catch (InterruptedException e) {
 +            log.warn("Interrupted waiting for " + nextNode.getClass().getSimpleName() + " to complete. Exiting.", e);
 +            break;
 +          } catch (ExecutionException e) {
 +            log.error("Caught error executing " + nextNode.getClass().getSimpleName(), e);
 +            throw e;
 +          } catch (TimeoutException e) {
 +            log.info("Timed out waiting for " + nextNode.getClass().getSimpleName() + " to complete (waited " + secondsRemaining + " seconds). Exiting.", e);
 +            break;
 +          }
 +
 +          // The RandomWalk node throw an Exception that that Callable handed back
 +          // Throw it and let the Module perform cleanup
 +          if (null != nodeException) {
 +            throw nodeException;
 +          }
 +
 +          if (test)
 +            stopTimer(nextNode);
 +        } catch (Exception e) {
 +          log.debug("Connector belongs to user: " + state.getConnector().whoami());
 +          log.debug("Exception occured at: " + System.currentTimeMillis());
 +          log.debug("Properties for node: " + nextNodeId);
 +          for (Entry<Object,Object> entry : nodeProps.entrySet()) {
 +            log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +          }
 +          log.debug("Overall Properties");
 +          for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
 +            log.debug("  " + entry.getKey() + ": " + entry.getValue());
 +          }
 +          log.debug("State information");
 +          for (String key : new TreeSet<String>(state.getMap().keySet())) {
 +            Object value = state.getMap().get(key);
 +            String logMsg = "  " + key + ": ";
 +            if (value == null)
 +              logMsg += "null";
 +            else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
 +              logMsg += value;
 +            else if (value instanceof byte[])
-               logMsg += new String((byte[]) value, Constants.UTF8);
++              logMsg += new String((byte[]) value, UTF_8);
 +            else if (value instanceof PasswordToken)
-               logMsg += new String(((PasswordToken) value).getPassword(), Constants.UTF8);
++              logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
 +            else
 +              logMsg += value.getClass() + " - " + value;
 +
 +            log.debug(logMsg);
 +          }
 +          throw new Exception("Error running node " + nextNodeId, e);
          }
 -      
 -      curNodeId = nextNodeId;
 +        state.visitedNode();
 +
 +        // update aliases
 +        if ((aliases = aliasMap.get(curNodeId)) != null)
 +          for (String alias : aliases) {
 +            ((Alias) nodes.get(alias)).update(curNodeId);
 +          }
 +
 +        curNodeId = nextNodeId;
 +      }
 +    } finally {
 +      if (null != service) {
 +        service.shutdownNow();
 +      }
      }
 -    
 +
      if (teardown && (fixture != null)) {
        log.debug("tearing down module");
        fixture.tearDown(state);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
index 1704e49,05cc4f0..67438d9
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
@@@ -20,10 -21,10 +21,10 @@@ import static com.google.common.base.Ch
  import org.apache.accumulo.core.data.Value;
  import org.apache.accumulo.test.randomwalk.State;
  
 -public class BulkMinusOne extends BulkTest {
 -  
 +public class BulkMinusOne extends BulkImportTest {
 +
-   private static final Value negOne = new Value("-1".getBytes(Constants.UTF8));
+   private static final Value negOne = new Value("-1".getBytes(UTF_8));
 -  
 +
    @Override
    protected void runLater(State state) throws Exception {
      log.info("Decrementing");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
index f002274,7f9a218..3dcff6e
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
@@@ -51,10 -52,10 +52,10 @@@ public class BatchWrite extends Test 
        try {
          int numRows = rand.nextInt(100000);
          for (int i = 0; i < numRows; i++) {
 -          Mutation m = new Mutation(String.format("%016x", (rand.nextLong() & 0x7fffffffffffffffl)));
 -          long val = (rand.nextLong() & 0x7fffffffffffffffl);
 +          Mutation m = new Mutation(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl));
 +          long val = rand.nextLong() & 0x7fffffffffffffffl;
            for (int j = 0; j < 10; j++) {
-             m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(Constants.UTF8)));
+             m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(UTF_8)));
            }
            
            bw.addMutation(m);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
index c8da3d8,c227a11..b3519b6
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
@@@ -47,8 -50,8 +49,8 @@@ public class StopTabletServer extends T
            Collections.sort(children);
            Stat stat = new Stat();
            byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
-           if (!"master".equals(new String(data, Constants.UTF8))) {
+           if (!"master".equals(new String(data, UTF_8))) {
 -            result.add(new TServerInstance(AddressUtil.parseAddress(child, Property.TSERV_CLIENTPORT), stat.getEphemeralOwner()));
 +            result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
            }
          }
        } catch (KeeperException.NoNodeException ex) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------


[41/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 103cfe8,0000000..547ccc7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@@ -1,1601 -1,0 +1,1603 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.nio.ByteBuffer;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.EnumSet;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +import java.util.Set;
 +import java.util.SortedSet;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.zip.ZipEntry;
 +import java.util.zip.ZipInputStream;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.NamespaceExistsException;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableDeletedException;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.TableOfflineException;
 +import org.apache.accumulo.core.client.admin.DiskUsage;
 +import org.apache.accumulo.core.client.admin.FindMax;
 +import org.apache.accumulo.core.client.admin.TableOperations;
 +import org.apache.accumulo.core.client.admin.TimeType;
 +import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService;
 +import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
 +import org.apache.accumulo.core.client.impl.thrift.TDiskUsage;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Constraint;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorUtil;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.master.thrift.FateOperation;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.metadata.MetadataServicer;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.util.ArgumentChecker;
 +import org.apache.accumulo.core.util.ByteBufferUtil;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.LocalityGroupUtil;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.core.util.NamingThreadFactory;
 +import org.apache.accumulo.core.util.OpTimer;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.StringUtil;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TApplicationException;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.transport.TTransportException;
 +
 +public class TableOperationsImpl extends TableOperationsHelper {
 +  private Instance instance;
 +  private Credentials credentials;
 +
 +  public static final String CLONE_EXCLUDE_PREFIX = "!";
 +
 +  private static final Logger log = Logger.getLogger(TableOperations.class);
 +
 +  /**
 +   * @param instance
 +   *          the connection information for this instance
 +   * @param credentials
 +   *          the username/password for this connection
 +   */
 +  public TableOperationsImpl(Instance instance, Credentials credentials) {
 +    ArgumentChecker.notNull(instance, credentials);
 +    this.instance = instance;
 +    this.credentials = credentials;
 +  }
 +
 +  /**
 +   * Retrieve a list of tables in Accumulo.
 +   *
 +   * @return List of tables in accumulo
 +   */
 +  @Override
 +  public SortedSet<String> list() {
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
 +    TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
 +    opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
 +    return tableNames;
 +  }
 +
 +  /**
 +   * A method to check if a table exists in Accumulo.
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @return true if the table exists
 +   */
 +  @Override
 +  public boolean exists(String tableName) {
 +    ArgumentChecker.notNull(tableName);
 +    if (tableName.equals(MetadataTable.NAME) || tableName.equals(RootTable.NAME))
 +      return true;
 +
 +    OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + " exists...");
 +    boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
 +    opTimer.stop("Checked existance of " + exists + " in %DURATION%");
 +    return exists;
 +  }
 +
 +  /**
 +   * Create a table with no special configuration
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableExistsException
 +   *           if the table already exists
 +   */
 +  @Override
 +  public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 +    create(tableName, true, TimeType.MILLIS);
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param limitVersion
 +   *          Enables/disables the versioning iterator, which will limit the number of Key versions kept.
 +   */
 +  @Override
 +  public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 +    create(tableName, limitVersion, TimeType.MILLIS);
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param timeType
 +   *          specifies logical or real-time based time recording for entries in the table
 +   * @param limitVersion
 +   *          Enables/disables the versioning iterator, which will limit the number of Key versions kept.
 +   */
 +  @Override
 +  public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
 +    ArgumentChecker.notNull(tableName, timeType);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(UTF_8)));
 +
 +    Map<String,String> opts;
 +    if (limitVersion)
 +      opts = IteratorUtil.generateInitialTableProperties(limitVersion);
 +    else
 +      opts = Collections.emptyMap();
 +
 +    try {
 +      doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_CREATE, args, opts);
 +    } catch (TableNotFoundException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  private long beginFateOperation() throws ThriftSecurityException, TException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        return client.beginFateOperation(Tracer.traceInfo(), credentials.toThrift(instance));
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call beginFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  // This method is for retrying in the case of network failures; anything else it passes to the caller to deal with
 +  private void executeFateOperation(long opid, FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
 +      throws ThriftSecurityException, TException, ThriftTableOperationException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        client.executeFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid, op, args, opts, autoCleanUp);
 +        break;
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call executeFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  private String waitForFateOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        return client.waitForFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call waitForFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  private void finishFateOperation(long opid) throws ThriftSecurityException, TException {
 +    while (true) {
 +      MasterClientService.Iface client = null;
 +      try {
 +        client = MasterClient.getConnectionWithRetry(instance);
 +        client.finishFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
 +        break;
 +      } catch (TTransportException tte) {
 +        log.debug("Failed to call finishFateOperation(), retrying ... ", tte);
 +        UtilWaitThread.sleep(100);
 +      } finally {
 +        MasterClient.close(client);
 +      }
 +    }
 +  }
 +
 +  String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
 +      TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
 +    return doFateOperation(op, args, opts, true);
 +  }
 +
 +  String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
 +      TableExistsException, TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
 +    Long opid = null;
 +
 +    try {
 +      opid = beginFateOperation();
 +      executeFateOperation(opid, op, args, opts, !wait);
 +      if (!wait) {
 +        opid = null;
 +        return null;
 +      }
 +      String ret = waitForFateOperation(opid);
 +      return ret;
 +    } catch (ThriftSecurityException e) {
 +      String tableName = ByteBufferUtil.toString(args.get(0));
 +      switch (e.getCode()) {
 +        case TABLE_DOESNT_EXIST:
 +          throw new TableNotFoundException(null, tableName, "Target table does not exist");
 +        case NAMESPACE_DOESNT_EXIST:
 +          throw new NamespaceNotFoundException(null, tableName, "Target namespace does not exist");
 +        default:
 +          String tableInfo = Tables.getPrintableTableInfoFromName(instance, tableName);
 +          throw new AccumuloSecurityException(e.user, e.code, tableInfo, e);
 +      }
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case EXISTS:
 +          throw new TableExistsException(e);
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        case NAMESPACE_EXISTS:
 +          throw new NamespaceExistsException(e);
 +        case NAMESPACE_NOTFOUND:
 +          throw new NamespaceNotFoundException(e);
 +        case OFFLINE:
 +          throw new TableOfflineException(instance, null);
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (Exception e) {
 +      throw new AccumuloException(e.getMessage(), e);
 +    } finally {
 +      Tables.clearCache(instance);
 +      // always finish table op, even when exception
 +      if (opid != null)
 +        try {
 +          finishFateOperation(opid);
 +        } catch (Exception e) {
 +          log.warn(e.getMessage(), e);
 +        }
 +    }
 +  }
 +
 +  private static class SplitEnv {
 +    private String tableName;
 +    private String tableId;
 +    private ExecutorService executor;
 +    private CountDownLatch latch;
 +    private AtomicReference<Exception> exception;
 +
 +    SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
 +      this.tableName = tableName;
 +      this.tableId = tableId;
 +      this.executor = executor;
 +      this.latch = latch;
 +      this.exception = exception;
 +    }
 +  }
 +
 +  private class SplitTask implements Runnable {
 +
 +    private List<Text> splits;
 +    private SplitEnv env;
 +
 +    SplitTask(SplitEnv env, List<Text> splits) {
 +      this.env = env;
 +      this.splits = splits;
 +    }
 +
 +    @Override
 +    public void run() {
 +      try {
 +        if (env.exception.get() != null)
 +          return;
 +
 +        if (splits.size() <= 2) {
 +          addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
 +          for (int i = 0; i < splits.size(); i++)
 +            env.latch.countDown();
 +          return;
 +        }
 +
 +        int mid = splits.size() / 2;
 +
 +        // split the middle split point to ensure that child task split different tablets and can therefore
 +        // run in parallel
 +        addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
 +        env.latch.countDown();
 +
 +        env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
 +        env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
 +
 +      } catch (Exception e) {
 +        env.exception.compareAndSet(null, e);
 +      }
 +    }
 +
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param partitionKeys
 +   *          a sorted set of row key values to pre-split the table on
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 +    String tableId = Tables.getTableId(instance, tableName);
 +
 +    List<Text> splits = new ArrayList<Text>(partitionKeys);
 +    // should be sorted because we copied from a sorted set, but that makes assumptions about
 +    // how the copy was done so resort to be sure.
 +    Collections.sort(splits);
 +
 +    CountDownLatch latch = new CountDownLatch(splits.size());
 +    AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
 +
 +    ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
 +    try {
 +      executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
 +
 +      while (!latch.await(100, TimeUnit.MILLISECONDS)) {
 +        if (exception.get() != null) {
 +          executor.shutdownNow();
 +          Exception excep = exception.get();
 +          if (excep instanceof TableNotFoundException)
 +            throw (TableNotFoundException) excep;
 +          else if (excep instanceof AccumuloException)
 +            throw (AccumuloException) excep;
 +          else if (excep instanceof AccumuloSecurityException)
 +            throw (AccumuloSecurityException) excep;
 +          else if (excep instanceof RuntimeException)
 +            throw (RuntimeException) excep;
 +          else
 +            throw new RuntimeException(excep);
 +        }
 +      }
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    } finally {
 +      executor.shutdown();
 +    }
 +  }
 +
 +  private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
 +      TableNotFoundException, AccumuloServerException {
 +    TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(tableId));
 +
 +    for (Text split : partitionKeys) {
 +      boolean successful = false;
 +      int attempt = 0;
 +      long locationFailures = 0;
 +
 +      while (!successful) {
 +
 +        if (attempt > 0)
 +          UtilWaitThread.sleep(100);
 +
 +        attempt++;
 +
 +        TabletLocation tl = tabLocator.locateTablet(credentials, split, false, false);
 +
 +        if (tl == null) {
 +          if (!Tables.exists(instance, tableId))
 +            throw new TableNotFoundException(tableId, tableName, null);
 +          else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
 +            throw new TableOfflineException(instance, tableId);
 +          continue;
 +        }
 +
 +        try {
 +          TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, ServerConfigurationUtil.getConfiguration(instance));
 +          try {
 +            OpTimer opTimer = null;
 +            if (log.isTraceEnabled())
 +              opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
 +
 +            client.splitTablet(Tracer.traceInfo(), credentials.toThrift(instance), tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
 +
 +            // just split it, might as well invalidate it in the cache
 +            tabLocator.invalidateCache(tl.tablet_extent);
 +
 +            if (opTimer != null)
 +              opTimer.stop("Split tablet in %DURATION%");
 +          } finally {
 +            ThriftUtil.returnClient(client);
 +          }
 +
 +        } catch (TApplicationException tae) {
 +          throw new AccumuloServerException(tl.tablet_location, tae);
 +        } catch (TTransportException e) {
 +          tabLocator.invalidateCache(tl.tablet_location);
 +          continue;
 +        } catch (ThriftSecurityException e) {
 +          Tables.clearCache(instance);
 +          if (!Tables.exists(instance, tableId))
 +            throw new TableNotFoundException(tableId, tableName, null);
 +          throw new AccumuloSecurityException(e.user, e.code, e);
 +        } catch (NotServingTabletException e) {
 +          // Do not silently spin when we repeatedly fail to get the location for a tablet
 +          locationFailures++;
 +          if (5 == locationFailures || 0 == locationFailures % 50) {
 +            log.warn("Having difficulty locating hosting tabletserver for split " + split + " on table " + tableName + ". Seen " + locationFailures
 +                + " failures.");
 +          }
 +
 +          tabLocator.invalidateCache(tl.tablet_extent);
 +          continue;
 +        } catch (TException e) {
 +          tabLocator.invalidateCache(tl.tablet_location);
 +          continue;
 +        }
 +
 +        successful = true;
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    ArgumentChecker.notNull(tableName);
 +    ByteBuffer EMPTY = ByteBuffer.allocate(0);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
 +        end == null ? EMPTY : TextUtil.getByteBuffer(end));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_MERGE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  @Override
 +  public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    ArgumentChecker.notNull(tableName);
 +    ByteBuffer EMPTY = ByteBuffer.allocate(0);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
 +        end == null ? EMPTY : TextUtil.getByteBuffer(end));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE_RANGE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @return the split points (end-row names) for the table's current split profile
 +   */
 +  @Override
 +  public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
 +
 +    ArgumentChecker.notNull(tableName);
 +
 +    String tableId = Tables.getTableId(instance, tableName);
 +
 +    TreeMap<KeyExtent,String> tabletLocations = new TreeMap<KeyExtent,String>();
 +
 +    while (true) {
 +      try {
 +        tabletLocations.clear();
 +        // the following method throws AccumuloException for some conditions that should be retried
 +        MetadataServicer.forTableId(instance, credentials, tableId).getTabletLocations(tabletLocations);
 +        break;
 +      } catch (AccumuloSecurityException ase) {
 +        throw ase;
 +      } catch (Exception e) {
 +        if (!Tables.exists(instance, tableId)) {
 +          throw new TableNotFoundException(tableId, tableName, null);
 +        }
 +
 +        if (e instanceof RuntimeException && e.getCause() instanceof AccumuloSecurityException) {
 +          throw (AccumuloSecurityException) e.getCause();
 +        }
 +
 +        log.info(e.getMessage() + " ... retrying ...");
 +        UtilWaitThread.sleep(3000);
 +      }
 +    }
 +
 +    ArrayList<Text> endRows = new ArrayList<Text>(tabletLocations.size());
 +
 +    for (KeyExtent ke : tabletLocations.keySet())
 +      if (ke.getEndRow() != null)
 +        endRows.add(ke.getEndRow());
 +
 +    return endRows;
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
 +    try {
 +      return listSplits(tableName);
 +    } catch (AccumuloSecurityException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param maxSplits
 +   *          specifies the maximum number of splits to return
 +   * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
 +   */
 +  @Override
 +  public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
 +    Collection<Text> endRows = listSplits(tableName);
 +
 +    if (endRows.size() <= maxSplits)
 +      return endRows;
 +
 +    double r = (maxSplits + 1) / (double) (endRows.size());
 +    double pos = 0;
 +
 +    ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
 +
 +    int j = 0;
 +    for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
 +      pos += r;
 +      while (pos > 1) {
 +        subset.add(((ArrayList<Text>) endRows).get(i));
 +        j++;
 +        pos -= 1;
 +      }
 +    }
 +
 +    return subset;
 +  }
 +
 +  @Deprecated
 +  @Override
 +  public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
 +    try {
 +      return listSplits(tableName, maxSplits);
 +    } catch (AccumuloSecurityException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  /**
 +   * Delete a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +  }
 +
 +  @Override
 +  public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
 +      throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
 +
 +    ArgumentChecker.notNull(srcTableName, newTableName);
 +
 +    String srcTableId = Tables.getTableId(instance, srcTableName);
 +
 +    if (flush)
 +      _flush(srcTableId, null, null, true);
 +
 +    if (propertiesToExclude == null)
 +      propertiesToExclude = Collections.emptySet();
 +
 +    if (propertiesToSet == null)
 +      propertiesToSet = Collections.emptyMap();
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    for (Entry<String,String> entry : propertiesToSet.entrySet()) {
 +      if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
 +        throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
 +      opts.put(entry.getKey(), entry.getValue());
 +    }
 +
 +    for (String prop : propertiesToExclude) {
 +      opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
 +    }
 +
 +    doTableFateOperation(newTableName, AccumuloException.class, FateOperation.TABLE_CLONE, args, opts);
 +  }
 +
 +  /**
 +   * Rename a table
 +   *
 +   * @param oldTableName
 +   *          the old table name
 +   * @param newTableName
 +   *          the new table name
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the old table name does not exist
 +   * @throws TableExistsException
 +   *           if the new table name already exists
 +   */
 +  @Override
 +  public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
 +      TableExistsException {
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +    doTableFateOperation(oldTableName, TableNotFoundException.class, FateOperation.TABLE_RENAME, args, opts);
 +  }
 +
 +  /**
 +   * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
 +   */
 +  @Override
 +  @Deprecated
 +  public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
 +    try {
 +      flush(tableName, null, null, false);
 +    } catch (TableNotFoundException e) {
 +      throw new AccumuloException(e.getMessage(), e);
 +    }
 +  }
 +
 +  /**
 +   * Flush a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   */
 +  @Override
 +  public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +
 +    String tableId = Tables.getTableId(instance, tableName);
 +    _flush(tableId, start, end, wait);
 +  }
 +
 +  @Override
 +  public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
 +      AccumuloException {
 +    compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
 +  }
 +
 +  @Override
 +  public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
 +      TableNotFoundException, AccumuloException {
 +    ArgumentChecker.notNull(tableName);
 +    ByteBuffer EMPTY = ByteBuffer.allocate(0);
 +
 +    String tableId = Tables.getTableId(instance, tableName);
 +
 +    if (flush)
 +      _flush(tableId, start, end, true);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
 +        end == null ? EMPTY : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
 +
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doFateOperation(FateOperation.TABLE_COMPACT, args, opts, wait);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    } catch (NamespaceExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    } catch (NamespaceNotFoundException e) {
 +      throw new TableNotFoundException(null, tableName, "Namespace not found", e);
 +    }
 +  }
 +
 +  @Override
 +  public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
 +    String tableId = Tables.getTableId(instance, tableName);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
 +
 +    Map<String,String> opts = new HashMap<String,String>();
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_CANCEL_COMPACT, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +  }
 +
 +  private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    try {
 +      long flushID;
 +
 +      // used to pass the table name. but the tableid associated with a table name could change between calls.
 +      // so pass the tableid to both calls
 +
 +      while (true) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnectionWithRetry(instance);
 +          flushID = client.initiateFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId);
 +          break;
 +        } catch (TTransportException tte) {
 +          log.debug("Failed to call initiateFlush, retrying ... ", tte);
 +          UtilWaitThread.sleep(100);
 +        } finally {
 +          MasterClient.close(client);
 +        }
 +      }
 +
 +      while (true) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnectionWithRetry(instance);
 +          client.waitForFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
 +              wait ? Long.MAX_VALUE : 1);
 +          break;
 +        } catch (TTransportException tte) {
 +          log.debug("Failed to call initiateFlush, retrying ... ", tte);
 +          UtilWaitThread.sleep(100);
 +        } finally {
 +          MasterClient.close(client);
 +        }
 +      }
 +    } catch (ThriftSecurityException e) {
 +      switch (e.getCode()) {
 +        case TABLE_DOESNT_EXIST:
 +          throw new TableNotFoundException(tableId, null, e.getMessage(), e);
 +        default:
 +          log.debug("flush security exception on table id " + tableId);
 +          throw new AccumuloSecurityException(e.user, e.code, e);
 +      }
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  /**
 +   * Sets a property on a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @param property
 +   *          the name of a per-table property
 +   * @param value
 +   *          the value to set a per-table property to
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   */
 +  @Override
 +  public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, property, value);
 +    try {
 +      MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
 +        @Override
 +        public void execute(MasterClientService.Client client) throws Exception {
 +          client.setTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property, value);
 +        }
 +      });
 +    } catch (TableNotFoundException e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  /**
 +   * Removes a property from a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @param property
 +   *          the name of a per-table property
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   */
 +  @Override
 +  public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, property);
 +    try {
 +      MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
 +        @Override
 +        public void execute(MasterClientService.Client client) throws Exception {
 +          client.removeTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property);
 +        }
 +      });
 +    } catch (TableNotFoundException e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  /**
 +   * Gets properties of a table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @return all properties visible by this table (system and per-table properties)
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +    try {
 +      return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
 +        @Override
 +        public Map<String,String> execute(ClientService.Client client) throws Exception {
 +          return client.getTableConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), tableName);
 +        }
 +      }).entrySet();
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        case NAMESPACE_NOTFOUND:
 +          throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (AccumuloException e) {
 +      throw e;
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    }
 +
 +  }
 +
 +  /**
 +   * Sets a tables locality groups. A tables locality groups can be changed at any time.
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @param groups
 +   *          mapping of locality group names to column families in the locality group
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    // ensure locality groups do not overlap
 +    HashSet<Text> all = new HashSet<Text>();
 +    for (Entry<String,Set<Text>> entry : groups.entrySet()) {
 +
 +      if (!Collections.disjoint(all, entry.getValue())) {
 +        throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
 +      }
 +
 +      all.addAll(entry.getValue());
 +    }
 +
 +    for (Entry<String,Set<Text>> entry : groups.entrySet()) {
 +      Set<Text> colFams = entry.getValue();
 +      String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
 +      setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
 +    }
 +
 +    try {
 +      setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
 +    } catch (AccumuloException e) {
 +      if (e.getCause() instanceof TableNotFoundException)
 +        throw (TableNotFoundException) e.getCause();
 +      throw e;
 +    }
 +
 +    // remove anything extraneous
 +    String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
 +    for (Entry<String,String> entry : getProperties(tableName)) {
 +      String property = entry.getKey();
 +      if (property.startsWith(prefix)) {
 +        // this property configures a locality group, find out which
 +        // one:
 +        String[] parts = property.split("\\.");
 +        String group = parts[parts.length - 1];
 +
 +        if (!groups.containsKey(group)) {
 +          removeProperty(tableName, property);
 +        }
 +      }
 +    }
 +  }
 +
 +  /**
 +   *
 +   * Gets the locality groups currently set for a table.
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @return mapping of locality group names to column families in the locality group
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
 +    AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
 +    Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
 +
 +    Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
 +    for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
 +
 +      HashSet<Text> colFams = new HashSet<Text>();
 +
 +      for (ByteSequence bs : entry.getValue()) {
 +        colFams.add(new Text(bs.toArray()));
 +      }
 +
 +      groups2.put(entry.getKey(), colFams);
 +    }
 +
 +    return groups2;
 +  }
 +
 +  /**
 +   * @param tableName
 +   *          the name of the table
 +   * @param range
 +   *          a range to split
 +   * @param maxSplits
 +   *          the maximum number of splits
 +   * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
 +   * @throws AccumuloException
 +   *           if a general error occurs
 +   * @throws AccumuloSecurityException
 +   *           if the user does not have permission
 +   * @throws TableNotFoundException
 +   *           if the table does not exist
 +   */
 +  @Override
 +  public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
 +      TableNotFoundException {
 +    ArgumentChecker.notNull(tableName, range);
 +    if (maxSplits < 1)
 +      throw new IllegalArgumentException("maximum splits must be >= 1");
 +    if (maxSplits == 1)
 +      return Collections.singleton(range);
 +
 +    Random random = new Random();
 +    Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
 +    String tableId = Tables.getTableId(instance, tableName);
 +    TabletLocator tl = TabletLocator.getLocator(instance, new Text(tableId));
 +    // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
 +    tl.invalidateCache();
 +    while (!tl.binRanges(credentials, Collections.singletonList(range), binnedRanges).isEmpty()) {
 +      if (!Tables.exists(instance, tableId))
 +        throw new TableDeletedException(tableId);
 +      if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
 +        throw new TableOfflineException(instance, tableId);
 +
 +      log.warn("Unable to locate bins for specified range. Retrying.");
 +      // sleep randomly between 100 and 200ms
 +      UtilWaitThread.sleep(100 + random.nextInt(100));
 +      binnedRanges.clear();
 +      tl.invalidateCache();
 +    }
 +
 +    // group key extents to get <= maxSplits
 +    LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
 +    List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
 +
 +    for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
 +      unmergedExtents.addAll(map.keySet());
 +
 +    // the sort method is efficient for linked list
 +    Collections.sort(unmergedExtents);
 +
 +    while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
 +      if (unmergedExtents.size() >= 2) {
 +        KeyExtent first = unmergedExtents.removeFirst();
 +        KeyExtent second = unmergedExtents.removeFirst();
 +        first.setEndRow(second.getEndRow());
 +        mergedExtents.add(first);
 +      } else {
 +        mergedExtents.addAll(unmergedExtents);
 +        unmergedExtents.clear();
 +        unmergedExtents.addAll(mergedExtents);
 +        mergedExtents.clear();
 +      }
 +
 +    }
 +
 +    mergedExtents.addAll(unmergedExtents);
 +
 +    Set<Range> ranges = new HashSet<Range>();
 +    for (KeyExtent k : mergedExtents)
 +      ranges.add(k.toDataRange().clip(range));
 +
 +    return ranges;
 +  }
 +
 +  // TODO Remove deprecation warning surppression when Hadoop1 support is dropped
 +  @SuppressWarnings("deprecation")
 +  private Path checkPath(String dir, String kind, String type) throws IOException, AccumuloException {
 +    Path ret;
 +    FileSystem fs = VolumeConfiguration.getVolume(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance)).getFileSystem();
 +
 +    if (dir.contains(":")) {
 +      ret = new Path(dir);
 +    } else {
 +      ret = fs.makeQualified(new Path(dir));
 +    }
 +
 +    if (!fs.exists(ret))
 +      throw new AccumuloException(kind + " import " + type + " directory " + dir + " does not exist!");
 +
 +    if (!fs.getFileStatus(ret).isDir()) {
 +      throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not a directory!");
 +    }
 +
 +    if (type.equals("failure")) {
 +      FileStatus[] listStatus = fs.listStatus(ret);
 +      if (listStatus != null && listStatus.length != 0) {
 +        throw new AccumuloException("Bulk import failure directory " + ret + " is not empty");
 +      }
 +    }
 +
 +    return ret;
 +  }
 +
 +  @Override
 +  public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
 +      TableNotFoundException, AccumuloException {
 +    ArgumentChecker.notNull(tableName, dir, failureDir);
 +    // check for table existance
 +    Tables.getTableId(instance, tableName);
 +
 +    Path dirPath = checkPath(dir, "Bulk", "");
 +    Path failPath = checkPath(failureDir, "Bulk", "failure");
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
-         ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(dirPath.toString().getBytes(UTF_8)),
++        ByteBuffer.wrap(failPath.toString().getBytes(UTF_8)), ByteBuffer.wrap((setTime + "").getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_BULK_IMPORT, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  private void waitForTableStateTransition(String tableId, TableState expectedState) throws AccumuloException, TableNotFoundException,
 +      AccumuloSecurityException {
 +
 +    Text startRow = null;
 +    Text lastRow = null;
 +
 +    while (true) {
 +
 +      if (Tables.getTableState(instance, tableId) != expectedState) {
 +        Tables.clearCache(instance);
 +        if (Tables.getTableState(instance, tableId) != expectedState) {
 +          if (!Tables.exists(instance, tableId))
 +            throw new TableDeletedException(tableId);
 +          throw new AccumuloException("Unexpected table state " + tableId + " " + Tables.getTableState(instance, tableId) + " != " + expectedState);
 +        }
 +      }
 +
 +      Range range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
 +      if (startRow == null || lastRow == null)
 +        range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
 +      else
 +        range = new Range(startRow, lastRow);
 +
 +      String metaTable = MetadataTable.NAME;
 +      if (tableId.equals(MetadataTable.ID))
 +        metaTable = RootTable.NAME;
 +      Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(metaTable, Authorizations.EMPTY);
 +      scanner = new IsolatedScanner(scanner);
 +      TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +      scanner.setRange(range);
 +
 +      RowIterator rowIter = new RowIterator(scanner);
 +
 +      KeyExtent lastExtent = null;
 +
 +      int total = 0;
 +      int waitFor = 0;
 +      int holes = 0;
 +      Text continueRow = null;
 +      MapCounter<String> serverCounts = new MapCounter<String>();
 +
 +      while (rowIter.hasNext()) {
 +        Iterator<Entry<Key,Value>> row = rowIter.next();
 +
 +        total++;
 +
 +        KeyExtent extent = null;
 +        String future = null;
 +        String current = null;
 +
 +        while (row.hasNext()) {
 +          Entry<Key,Value> entry = row.next();
 +          Key key = entry.getKey();
 +
 +          if (key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME))
 +            future = entry.getValue().toString();
 +
 +          if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME))
 +            current = entry.getValue().toString();
 +
 +          if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key))
 +            extent = new KeyExtent(key.getRow(), entry.getValue());
 +        }
 +
 +        if ((expectedState == TableState.ONLINE && current == null) || (expectedState == TableState.OFFLINE && (future != null || current != null))) {
 +          if (continueRow == null)
 +            continueRow = extent.getMetadataEntry();
 +          waitFor++;
 +          lastRow = extent.getMetadataEntry();
 +
 +          if (current != null)
 +            serverCounts.increment(current, 1);
 +          if (future != null)
 +            serverCounts.increment(future, 1);
 +        }
 +
 +        if (!extent.getTableId().toString().equals(tableId)) {
 +          throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
 +        }
 +
 +        if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
 +          holes++;
 +        }
 +
 +        lastExtent = extent;
 +      }
 +
 +      if (continueRow != null) {
 +        startRow = continueRow;
 +      }
 +
 +      if (holes > 0 || total == 0) {
 +        startRow = null;
 +        lastRow = null;
 +      }
 +
 +      if (waitFor > 0 || holes > 0 || total == 0) {
 +        long waitTime;
 +        long maxPerServer = 0;
 +        if (serverCounts.size() > 0) {
 +          maxPerServer = Collections.max(serverCounts.values());
 +          waitTime = maxPerServer * 10;
 +        } else
 +          waitTime = waitFor * 10;
 +        waitTime = Math.max(100, waitTime);
 +        waitTime = Math.min(5000, waitTime);
 +        log.trace("Waiting for " + waitFor + "(" + maxPerServer + ") tablets, startRow = " + startRow + " lastRow = " + lastRow + ", holes=" + holes
 +            + " sleeping:" + waitTime + "ms");
 +        UtilWaitThread.sleep(waitTime);
 +      } else {
 +        break;
 +      }
 +
 +    }
 +  }
 +
 +  @Override
 +  public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    offline(tableName, false);
 +  }
 +
 +  /**
 +   *
 +   * @param tableName
 +   *          the table to take offline
 +   * @throws AccumuloException
 +   *           when there is a general accumulo error
 +   * @throws AccumuloSecurityException
 +   *           when the user does not have the proper permissions
 +   */
 +  @Override
 +  public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +
 +    ArgumentChecker.notNull(tableName);
 +    String tableId = Tables.getTableId(instance, tableName);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_OFFLINE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +    if (wait)
 +      waitForTableStateTransition(tableId, TableState.OFFLINE);
 +  }
 +
 +  @Override
 +  public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    online(tableName, false);
 +  }
 +
 +  /**
 +   *
 +   * @param tableName
 +   *          the table to take online
 +   * @throws AccumuloException
 +   *           when there is a general accumulo error
 +   * @throws AccumuloSecurityException
 +   *           when the user does not have the proper permissions
 +   */
 +  @Override
 +  public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +    String tableId = Tables.getTableId(instance, tableName);
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
 +    Map<String,String> opts = new HashMap<String,String>();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_ONLINE, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +    if (wait)
 +      waitForTableStateTransition(tableId, TableState.ONLINE);
 +  }
 +
 +  /**
 +   * Clears the tablet locator cache for a specified table
 +   *
 +   * @param tableName
 +   *          the name of the table
 +   * @throws TableNotFoundException
 +   *           if table does not exist
 +   */
 +  @Override
 +  public void clearLocatorCache(String tableName) throws TableNotFoundException {
 +    ArgumentChecker.notNull(tableName);
 +    TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(Tables.getTableId(instance, tableName)));
 +    tabLocator.invalidateCache();
 +  }
 +
 +  /**
 +   * Get a mapping of table name to internal table id.
 +   *
 +   * @return the map from table name to internal table id
 +   */
 +  @Override
 +  public Map<String,String> tableIdMap() {
 +    return Tables.getNameToIdMap(instance);
 +  }
 +
 +  @Override
 +  public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
 +      throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, auths);
 +    Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, auths);
 +    return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
 +  }
 +
 +  @Override
 +  public List<DiskUsage> getDiskUsage(Set<String> tableNames) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +
 +    List<TDiskUsage> diskUsages = null;
 +    while (diskUsages == null) {
 +      Pair<String,Client> pair = null;
 +      try {
 +        // this operation may us a lot of memory... its likely that connections to tabletservers hosting metadata tablets will be cached, so do not use cached
 +        // connections
 +        pair = ServerClient.getConnection(instance, false);
 +        diskUsages = pair.getSecond().getDiskUsage(tableNames, credentials.toThrift(instance));
 +      } catch (ThriftTableOperationException e) {
 +        switch (e.getType()) {
 +          case NOTFOUND:
 +            throw new TableNotFoundException(e);
 +          case NAMESPACE_NOTFOUND:
 +            throw new TableNotFoundException(e.getTableName(), new NamespaceNotFoundException(e));
 +          default:
 +            throw new AccumuloException(e.description, e);
 +        }
 +      } catch (ThriftSecurityException e) {
 +        throw new AccumuloSecurityException(e.getUser(), e.getCode());
 +      } catch (TTransportException e) {
 +        // some sort of communication error occurred, retry
 +	if (pair == null) {
 +          log.debug("Disk usage request failed.  Pair is null.  Retrying request...", e);
 +	} else {
 +          log.debug("Disk usage request failed " + pair.getFirst() + ", retrying ... ", e);
 +	}
 +        UtilWaitThread.sleep(100);
 +      } catch (TException e) {
 +        // may be a TApplicationException which indicates error on the server side
 +        throw new AccumuloException(e);
 +      } finally {
 +        // must always return thrift connection
 +        if (pair != null)
 +          ServerClient.close(pair.getSecond());
 +      }
 +    }
 +
 +    List<DiskUsage> finalUsages = new ArrayList<DiskUsage>();
 +    for (TDiskUsage diskUsage : diskUsages) {
 +      finalUsages.add(new DiskUsage(new TreeSet<String>(diskUsage.getTables()), diskUsage.getUsage()));
 +    }
 +
 +    return finalUsages;
 +  }
 +
 +  public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
 +    HashMap<String,String> props = new HashMap<String,String>();
 +
 +    ZipInputStream zis = new ZipInputStream(fs.open(path));
 +    try {
 +      ZipEntry zipEntry;
 +      while ((zipEntry = zis.getNextEntry()) != null) {
 +        if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
-           BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++          BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
 +          try {
 +            String line;
 +            while ((line = in.readLine()) != null) {
 +              String sa[] = line.split("=", 2);
 +              props.put(sa[0], sa[1]);
 +            }
 +          } finally {
 +            in.close();
 +          }
 +
 +          break;
 +        }
 +      }
 +    } finally {
 +      zis.close();
 +    }
 +    return props;
 +  }
 +
 +  @Override
 +  public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, importDir);
 +
 +    try {
 +      importDir = checkPath(importDir, "Table", "").toString();
 +    } catch (IOException e) {
 +      throw new AccumuloException(e);
 +    }
 +
 +    try {
 +      FileSystem fs = new Path(importDir).getFileSystem(CachedConfiguration.getInstance());
 +      Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
 +
 +      for (Entry<String,String> entry : props.entrySet()) {
 +        if (Property.isClassProperty(entry.getKey()) && !entry.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
 +          Logger.getLogger(this.getClass()).info(
 +              "Imported table sets '" + entry.getKey() + "' to '" + entry.getValue() + "'.  Ensure this class is on Accumulo classpath.");
 +        }
 +      }
 +
 +    } catch (IOException ioe) {
 +      Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
 +    }
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(importDir.getBytes(UTF_8)));
 +
 +    Map<String,String> opts = Collections.emptyMap();
 +
 +    try {
 +      doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_IMPORT, args, opts);
 +    } catch (TableNotFoundException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +
 +  }
 +
 +  @Override
 +  public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, exportDir);
 +
-     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
++    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
 +
 +    Map<String,String> opts = Collections.emptyMap();
 +
 +    try {
 +      doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_EXPORT, args, opts);
 +    } catch (TableExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    }
 +  }
 +
 +  @Override
 +  public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
 +      AccumuloSecurityException {
 +    ArgumentChecker.notNull(tableName, className, asTypeName);
 +
 +    try {
 +      return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
 +        @Override
 +        public Boolean execute(ClientService.Client client) throws Exception {
 +          return client.checkTableClass(Tracer.traceInfo(), credentials.toThrift(instance), tableName, className, asTypeName);
 +        }
 +      });
 +    } catch (ThriftTableOperationException e) {
 +      switch (e.getType()) {
 +        case NOTFOUND:
 +          throw new TableNotFoundException(e);
 +        case NAMESPACE_NOTFOUND:
 +          throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
 +        default:
 +          throw new AccumuloException(e.description, e);
 +      }
 +    } catch (ThriftSecurityException e) {
 +      throw new AccumuloSecurityException(e.user, e.code, e);
 +    } catch (AccumuloException e) {
 +      throw e;
 +    } catch (Exception e) {
 +      throw new AccumuloException(e);
 +    }
 +  }
 +
 +  @Override
 +  public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
 +      TableNotFoundException {
 +    testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
 +    super.attachIterator(tableName, setting, scopes);
 +  }
 +
 +  @Override
 +  public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
 +    testClassLoad(tableName, constraintClassName, Constraint.class.getName());
 +    return super.addConstraint(tableName, constraintClassName);
 +  }
 +
 +  private void doTableFateOperation(String tableName, Class<? extends Exception> namespaceNotFoundExceptionClass, FateOperation op, List<ByteBuffer> args,
 +      Map<String,String> opts) throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
 +    try {
 +      doFateOperation(op, args, opts);
 +    } catch (NamespaceExistsException e) {
 +      // should not happen
 +      throw new AssertionError(e);
 +    } catch (NamespaceNotFoundException e) {
 +      if (namespaceNotFoundExceptionClass == null) {
 +        // should not happen
 +        throw new AssertionError(e);
 +      } else if (AccumuloException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
 +        throw new AccumuloException("Cannot create table in non-existent namespace", e);
 +      } else if (TableNotFoundException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
 +        throw new TableNotFoundException(null, tableName, "Namespace not found", e);
 +      } else {
 +        // should not happen
 +        throw new AssertionError(e);
 +      }
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
index 32b6e28,6414193..4de95c3
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
@@@ -16,8 -16,9 +16,10 @@@
   */
  package org.apache.accumulo.core.client.impl;
  
+ import static com.google.common.base.Charsets.UTF_8;
+ 
  import java.security.SecurityPermission;
 +import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.SortedMap;
@@@ -118,75 -41,36 +120,75 @@@ public class Tables 
      if (sm != null) {
        sm.checkPermission(TABLES_PERMISSION);
      }
 -    return ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
    }
 -  
 +
    private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
      ZooCache zc = getZooCache(instance);
 -    
 +
      List<String> tableIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES);
 -    
      TreeMap<String,String> tableMap = new TreeMap<String,String>();
 -    
 +    Map<String,String> namespaceIdToNameMap = new HashMap<String,String>();
 +
      for (String tableId : tableIds) {
 -      byte[] tblPath = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
 -      if (tblPath != null) {
 +      byte[] tableName = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
 +      byte[] nId = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
 +      String namespaceName = Namespaces.DEFAULT_NAMESPACE;
 +      // create fully qualified table name
 +      if (nId == null) {
 +        namespaceName = null;
 +      } else {
-         String namespaceId = new String(nId, Constants.UTF8);
++        String namespaceId = new String(nId, UTF_8);
 +        if (!namespaceId.equals(Namespaces.DEFAULT_NAMESPACE_ID)) {
 +          try {
 +            namespaceName = namespaceIdToNameMap.get(namespaceId);
 +            if (namespaceName == null) {
 +              namespaceName = Namespaces.getNamespaceName(instance, namespaceId);
 +              namespaceIdToNameMap.put(namespaceId, namespaceName);
 +            }
 +          } catch (NamespaceNotFoundException e) {
 +            log.error("Table (" + tableId + ") contains reference to namespace (" + namespaceId + ") that doesn't exist", e);
 +            continue;
 +          }
 +        }
 +      }
 +      if (tableName != null && namespaceName != null) {
-         String tableNameStr = qualified(new String(tableName, Constants.UTF8), namespaceName);
++        String tableNameStr = qualified(new String(tableName, UTF_8), namespaceName);
          if (nameAsKey)
 -          tableMap.put(new String(tblPath, UTF_8), tableId);
 +          tableMap.put(tableNameStr, tableId);
          else
 -          tableMap.put(tableId, new String(tblPath, UTF_8));
 +          tableMap.put(tableId, tableNameStr);
        }
      }
 -    
 +
      return tableMap;
    }
 -  
 +
    public static String getTableId(Instance instance, String tableName) throws TableNotFoundException {
 +    try {
 +      return _getTableId(instance, tableName);
 +    } catch (NamespaceNotFoundException e) {
 +      throw new TableNotFoundException(tableName, e);
 +    }
 +  }
 +
 +  public static String _getTableId(Instance instance, String tableName) throws NamespaceNotFoundException, TableNotFoundException {
      String tableId = getNameToIdMap(instance).get(tableName);
 -    if (tableId == null)
 -      throw new TableNotFoundException(tableId, tableName, null);
 +    if (tableId == null) {
 +      // maybe the table exist, but the cache was not updated yet... so try to clear the cache and check again
 +      clearCache(instance);
 +      tableId = getNameToIdMap(instance).get(tableName);
 +      if (tableId == null) {
 +        String namespace = qualify(tableName).getFirst();
 +        if (Namespaces.getNameToIdMap(instance).containsKey(namespace))
 +          throw new TableNotFoundException(null, tableName, null);
 +        else
 +          throw new NamespaceNotFoundException(null, namespace, null);
 +      }
 +    }
      return tableId;
    }
 -  
 +
    public static String getTableName(Instance instance, String tableId) throws TableNotFoundException {
      String tableName = getIdToNameMap(instance).get(tableId);
      if (tableName == null)
@@@ -250,65 -113,11 +252,65 @@@
      byte[] state = zc.get(statePath);
      if (state == null)
        return TableState.UNKNOWN;
 -    
 +
-     return TableState.valueOf(new String(state, Constants.UTF8));
+     return TableState.valueOf(new String(state, UTF_8));
    }
 -  
 +
    public static long getCacheResetCount() {
      return cacheResetCount.get();
    }
 +
 +  public static String qualified(String tableName) {
 +    return qualified(tableName, Namespaces.DEFAULT_NAMESPACE);
 +  }
 +
 +  public static String qualified(String tableName, String defaultNamespace) {
 +    Pair<String,String> qualifiedTableName = Tables.qualify(tableName, defaultNamespace);
 +    if (Namespaces.DEFAULT_NAMESPACE.equals(qualifiedTableName.getFirst()))
 +      return qualifiedTableName.getSecond();
 +    else
 +      return qualifiedTableName.toString("", ".", "");
 +  }
 +
 +  public static Pair<String,String> qualify(String tableName) {
 +    return qualify(tableName, Namespaces.DEFAULT_NAMESPACE);
 +  }
 +
 +  public static Pair<String,String> qualify(String tableName, String defaultNamespace) {
 +    if (!tableName.matches(Tables.VALID_NAME_REGEX))
 +      throw new IllegalArgumentException("Invalid table name '" + tableName + "'");
 +    if (MetadataTable.OLD_NAME.equals(tableName))
 +      tableName = MetadataTable.NAME;
 +    if (tableName.contains(".")) {
 +      String[] s = tableName.split("\\.", 2);
 +      return new Pair<String,String>(s[0], s[1]);
 +    }
 +    return new Pair<String,String>(defaultNamespace, tableName);
 +  }
 +
 +  /**
 +   * Returns the namespace id for a given table ID.
 +   * 
 +   * @param instance
 +   *          The Accumulo Instance
 +   * @param tableId
 +   *          The tableId
 +   * @return The namespace id which this table resides in.
 +   * @throws IllegalArgumentException
 +   *           if the table doesn't exist in ZooKeeper
 +   */
 +  public static String getNamespaceId(Instance instance, String tableId) throws IllegalArgumentException {
 +    ArgumentChecker.notNull(instance, tableId);
 +
 +    ZooCache zc = getZooCache(instance);
 +    byte[] n = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
 +
 +    // We might get null out of ZooCache if this tableID doesn't exist
 +    if (null == n) {
 +      throw new IllegalArgumentException("Table with id " + tableId + " does not exist");
 +    }
 +
-     return new String(n, Constants.UTF8);
++    return new String(n, UTF_8);
 +  }
 +
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index d58d1e4,a9f5b61..de7bcb2
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@@ -81,10 -82,10 +82,10 @@@ public class Writer 
        throw new IllegalArgumentException("Can not add empty mutations");
      
      while (true) {
 -      TabletLocation tabLoc = TabletLocator.getInstance(instance, table).locateTablet(new Text(m.getRow()), false, true, credentials);
 +      TabletLocation tabLoc = TabletLocator.getLocator(instance, table).locateTablet(credentials, new Text(m.getRow()), false, true);
        
        if (tabLoc == null) {
-         log.trace("No tablet location found for row " + new String(m.getRow(), Constants.UTF8));
+         log.trace("No tablet location found for row " + new String(m.getRow(), UTF_8));
          UtilWaitThread.sleep(500);
          continue;
        }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
index a071d07,0000000..bf365ef
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
@@@ -1,40 -1,0 +1,40 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.lexicoder;
 +
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
 +
 +/**
 + * This lexicoder encodes/decodes a given String to/from bytes without further processing. It can be combined with other encoders like the
 + * {@link ReverseLexicoder} to flip the default sort order.
 + * 
 + * @since 1.6.0
 + */
 +
 +public class StringLexicoder implements Lexicoder<String> {
 +  
 +  @Override
 +  public byte[] encode(String data) {
-     return data.getBytes(Constants.UTF8);
++    return data.getBytes(UTF_8);
 +  }
 +  
 +  @Override
 +  public String decode(byte[] data) {
-     return new String(data, Constants.UTF8);
++    return new String(data, UTF_8);
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 500047c,23540f2..563610a
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -27,8 -27,7 +29,7 @@@ import java.util.HashSet
  import java.util.List;
  import java.util.Set;
  
- import org.apache.accumulo.core.Constants;
 -import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientConfiguration;
  import org.apache.accumulo.core.client.Instance;
  import org.apache.accumulo.core.client.IteratorSetting;
  import org.apache.accumulo.core.client.ZooKeeperInstance;
@@@ -203,24 -189,14 +204,24 @@@ public class RangeInputSplit extends In
      }
  
      if (in.readBoolean()) {
 -      String tokenClass = in.readUTF();
 -      byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
 -      byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
 -
 -      try {
 -        token = CredentialHelper.extractToken(tokenClass, tokenBytes);
 -      } catch (AccumuloSecurityException e) {
 -        throw new IOException(e);
 +      int ordinal = in.readInt();
 +      this.tokenSource = TokenSource.values()[ordinal];
 +
 +      switch (this.tokenSource) {
 +        case INLINE:
 +          String tokenClass = in.readUTF();
-           byte[] base64TokenBytes = in.readUTF().getBytes(Constants.UTF8);
++          byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
 +          byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
 +
 +          this.token = AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes);
 +          break;
 +
 +        case FILE:
 +          this.tokenFile = in.readUTF();
 +
 +          break;
 +        default:
 +          throw new IOException("Cannot parse unknown TokenSource ordinal");
        }
      }
  


[47/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 7f2048b,7a1e1ba..c93f680
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index 3543a16,809a7c8..0652cf1
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -16,9 -16,10 +16,9 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
+ import java.io.IOException;
 -
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.cli.Help;
  import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index f728fec,d40830f..0793c6b
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.HashSet;
  import java.util.Set;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 565eaba,43348df..478b3b5
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -16,10 -16,11 +16,10 @@@
   */
  package org.apache.accumulo.server.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.PrintStream;
  import java.io.UnsupportedEncodingException;
- import java.nio.charset.StandardCharsets;
--
  import org.apache.accumulo.core.cli.Help;
  import org.apache.accumulo.core.util.Base64;
  import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f73747d,be85b1b..8550d0b
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Formattable;
  import java.util.Formatter;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index 05f28d7,e936b97..01eb477
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@@ -16,8 -16,7 +16,9 @@@
   */
  package org.apache.accumulo.server.util;
  
++import static java.nio.charset.StandardCharsets.UTF_8;
++
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.List;
@@@ -70,8 -69,8 +71,8 @@@ public class MasterMetadataUtil 
        Map<FileRef,Long> bulkLoadedFiles, Credentials credentials, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
      Mutation m = extent.getPrevRowUpdateMutation();
      
-     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(StandardCharsets.UTF_8)));
-     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(StandardCharsets.UTF_8)));
 -    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes()));
 -    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes()));
++    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
++    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
      if (lastFlushID > 0)
        TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(("" + lastFlushID).getBytes()));
      if (lastCompactID > 0)
@@@ -104,7 -103,7 +105,7 @@@
        throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")");
      }
      
-     double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get(), StandardCharsets.UTF_8));
 -    double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get()));
++    double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get(), UTF_8));
      
      Value prevEndRowIBW = columns.get(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
      
@@@ -245,62 -244,41 +246,62 @@@
        Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
      if (extent.isRootTablet()) {
        if (unusedWalLogs != null) {
 -        IZooReaderWriter zk = ZooReaderWriter.getInstance();
 -        // unusedWalLogs will contain the location/name of each log in a log set
 -        // the log set is stored under one of the log names, but not both
 -        // find the entry under one of the names and delete it.
 -        String root = MetadataTableUtil.getZookeeperLogLocation();
 -        boolean foundEntry = false;
 -        for (String entry : unusedWalLogs) {
 -          String[] parts = entry.split("/");
 -          String zpath = root + "/" + parts[parts.length - 1];
 -          while (true) {
 -            try {
 -              if (zk.exists(zpath)) {
 -                zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
 -                foundEntry = true;
 -              }
 -              break;
 -            } catch (KeeperException e) {
 -              log.error(e, e);
 -            } catch (InterruptedException e) {
 -              log.error(e, e);
 -            }
 -            UtilWaitThread.sleep(1000);
 +        updateRootTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId);
 +      }
 +
 +      return;
 +    }
 +
 +    Mutation m = getUpdateForTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId);
 +
 +    MetadataTableUtil.update(credentials, zooLock, m, extent);
 +
 +  }
 +
 +  /**
 +   * Update the data file for the root tablet
 +   */
 +  protected static void updateRootTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time,
 +      Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +    // unusedWalLogs will contain the location/name of each log in a log set
 +    // the log set is stored under one of the log names, but not both
 +    // find the entry under one of the names and delete it.
 +    String root = MetadataTableUtil.getZookeeperLogLocation();
 +    boolean foundEntry = false;
 +    for (String entry : unusedWalLogs) {
 +      String[] parts = entry.split("/");
 +      String zpath = root + "/" + parts[parts.length - 1];
 +      while (true) {
 +        try {
 +          if (zk.exists(zpath)) {
 +            zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
 +            foundEntry = true;
            }
 +          break;
 +        } catch (KeeperException e) {
 +          log.error(e, e);
 +        } catch (InterruptedException e) {
 +          log.error(e, e);
          }
 -        if (unusedWalLogs.size() > 0 && !foundEntry)
 -          log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
 +        UtilWaitThread.sleep(1000);
        }
 -      return;
      }
 -    
 +    if (unusedWalLogs.size() > 0 && !foundEntry)
 +      log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
 +  }
 +
 +  /**
 +   * Create an update that updates a tablet
 +   * @return A Mutation to update a tablet from the given information
 +   */
 +  protected static Mutation getUpdateForTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time,
 +      Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
      Mutation m = new Mutation(extent.getMetadataEntry());
 -    
 +
      if (dfv.getNumEntries() > 0) {
        m.put(DataFileColumnFamily.NAME, path.meta(), new Value(dfv.encode()));
-       TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(StandardCharsets.UTF_8)));
 -      TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes()));
++      TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
        // stuff in this location
        TServerInstance self = getTServerInstance(address, zooLock);
        self.putLastLocation(m);
@@@ -313,15 -291,17 +314,15 @@@
          m.putDelete(LogColumnFamily.NAME, new Text(entry));
        }
      }
 -    
 +
      for (FileRef scanFile : filesInUseByScans)
 -      m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value("".getBytes()));
 -    
 +      m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0]));
 +
      if (mergeFile != null)
        m.putDelete(DataFileColumnFamily.NAME, mergeFile.meta());
 -    
 -    TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushId + "").getBytes()));
 -    
 -    MetadataTableUtil.update(credentials, zooLock, m, extent);
 -    
 +
-     TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(StandardCharsets.UTF_8)));
++    TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(UTF_8)));
 +
 +    return m;
    }
 -  
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index cb97d2a,9cd02eb..8f84169
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.Comparator;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index 3bd7b8e,c0d6894..85cc484
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.conf.Property;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 9f298af,8924c85..d797392
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -16,12 -16,12 +16,12 @@@
   */
  package org.apache.accumulo.server.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
  
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Map.Entry;
  
 -import org.apache.accumulo.server.security.SystemCredentials;
  import org.apache.accumulo.core.cli.ClientOnRequiredTable;
  import org.apache.accumulo.core.client.AccumuloException;
  import org.apache.accumulo.core.client.AccumuloSecurityException;
@@@ -105,10 -105,9 +105,10 @@@ public class RandomizeVolumes 
        }
        Key key = entry.getKey();
        Mutation m = new Mutation(key.getRow());
 -      
 -      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
 +
 +      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
 +          + directory;
-       m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(StandardCharsets.UTF_8)));
+       m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
        if (log.isTraceEnabled()) {
          log.trace("Replacing " + oldLocation + " with " + newLocation);
        }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 3a8eb0b,a82fa14..b08bf90
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.server.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.FileInputStream;
  import java.io.InputStream;
- import java.nio.charset.StandardCharsets;
  import java.util.Stack;
  
  import javax.xml.parsers.SAXParser;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index ee01bb2,b1a553d..63f1343
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.util;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.FileInputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index 9543440,401d98c..a376ed6
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.conf.Property;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index a755c0a,6ee1593..5e6542d
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.conf.Property;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index f334613,edbb103..307bb0c
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.List;
  
  import org.apache.accumulo.core.Constants;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index ee20006,7ae4d5a..103aefc
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.watcher;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.zookeeper.ZooUtil;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
index 0f1a005,c5a9528..1452aa7
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
@@@ -16,7 -16,6 +16,8 @@@
   */
  package org.apache.accumulo.server.zookeeper;
  
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.List;
@@@ -212,14 -200,7 +213,14 @@@ public class DistributedWorkQueue 
            log.info("Interrupted looking for work", e);
          }
        }
 -    }, r.nextInt(60 * 1000), 60 * 1000);
 +    }, timerInitialDelay, timerPeriod);
 +  }
 +
 +  /**
-    * Adds work to the queue, automatically converting the String to bytes using {@link StandardCharsets#UTF_8}
++   * Adds work to the queue, automatically converting the String to bytes using UTF-8
 +   */
 +  public void addWork(String workId, String data) throws KeeperException, InterruptedException {
-     addWork(workId, data.getBytes(StandardCharsets.UTF_8));
++    addWork(workId, data.getBytes(UTF_8));
    }
    
    public void addWork(String workId, byte[] data) throws KeeperException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
index 9a968a9,1bbae03..e4ef1eb
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.zookeeper;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.locks.Lock;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
index e6d2350,c5374a2..34c9070
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.zookeeper;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.conf.AccumuloConfiguration;
  import org.apache.accumulo.core.conf.Property;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index e07c2b9,e15e63e..12bdeb1
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.zookeeper;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.conf.AccumuloConfiguration;
  import org.apache.accumulo.core.conf.Property;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index 36ddd3b,3aed7ef..628b981
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@@ -16,7 -16,8 +16,18 @@@
   */
  package org.apache.accumulo.server.conf;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
++import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertNull;
++import static org.junit.Assert.assertTrue;
++import static org.easymock.EasyMock.anyObject;
++import static org.easymock.EasyMock.createMock;
++import static org.easymock.EasyMock.eq;
++import static org.easymock.EasyMock.expect;
++import static org.easymock.EasyMock.expectLastCall;
++import static org.easymock.EasyMock.replay;
++import static org.easymock.EasyMock.verify;
+ 
  import java.util.Collection;
  import java.util.List;
  import java.util.Map;
@@@ -34,16 -35,16 +45,6 @@@ import org.apache.accumulo.fate.zookeep
  import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
  import org.junit.Before;
  import org.junit.Test;
--import static org.junit.Assert.assertEquals;
--import static org.junit.Assert.assertNull;
--import static org.junit.Assert.assertTrue;
--import static org.easymock.EasyMock.anyObject;
--import static org.easymock.EasyMock.createMock;
--import static org.easymock.EasyMock.eq;
--import static org.easymock.EasyMock.expect;
--import static org.easymock.EasyMock.expectLastCall;
--import static org.easymock.EasyMock.replay;
--import static org.easymock.EasyMock.verify;
  
  public class NamespaceConfigurationTest {
    private static final String NSID = "namespace";

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
index 475afbb,81148a0..cec75c2
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.conf;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.easymock.EasyMock.anyObject;
  import static org.easymock.EasyMock.createMock;
  import static org.easymock.EasyMock.endsWith;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
index 12521f9,2f38e4c..e66ec98
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.conf;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.easymock.EasyMock.anyObject;
  import static org.easymock.EasyMock.createMock;
  import static org.easymock.EasyMock.eq;
@@@ -84,8 -84,8 +84,8 @@@ public class TableConfigurationTest 
    @Test
    public void testGet_InZK() {
      Property p = Property.INSTANCE_SECRET;
 -    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
 -        .andReturn("sekrit".getBytes(UTF_8));
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(
-         "sekrit".getBytes(StandardCharsets.UTF_8));
++        "sekrit".getBytes(UTF_8));
      replay(zc);
      assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
    }
@@@ -110,10 -110,8 +110,10 @@@
      children.add("foo");
      children.add("ding");
      expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF)).andReturn(children);
 -    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(UTF_8));
 -    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(UTF_8));
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo"))
-         .andReturn("bar".getBytes(StandardCharsets.UTF_8));
++        .andReturn("bar".getBytes(UTF_8));
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn(
-         "dong".getBytes(StandardCharsets.UTF_8));
++        "dong".getBytes(UTF_8));
      replay(zc);
      c.getProperties(props, filter);
      assertEquals(2, props.size());
@@@ -137,8 -135,8 +137,8 @@@
    public void testInvalidateCache() {
      // need to do a get so the accessor is created
      Property p = Property.INSTANCE_SECRET;
 -    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
 -        .andReturn("sekrit".getBytes(UTF_8));
 +    expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(
-         "sekrit".getBytes(StandardCharsets.UTF_8));
++        "sekrit".getBytes(UTF_8));
      zc.clear();
      replay(zc);
      c.get(Property.INSTANCE_SECRET);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
index 742ccdd,4369119..210d240
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.conf;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.easymock.EasyMock.createMock;
  import static org.easymock.EasyMock.expect;
  import static org.easymock.EasyMock.replay;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index e5f3955,83e83cf..e3fc69d
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.master;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;
@@@ -301,53 -291,20 +302,53 @@@ public class Master implements LiveTSer
          IZooReaderWriter zoo = ZooReaderWriter.getInstance();
          final String zooRoot = ZooUtil.getRoot(instance);
  
 -        if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
 -          log.debug("Handling updates for version " + ServerConstants.TWO_DATA_VERSIONS_AGO);
 +        log.debug("Handling updates for version " + accumuloPersistentVersion);
  
 -          log.debug("Cleaning out remnants of logger role.");
 -          zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
 -          zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
 +        log.debug("Cleaning out remnants of logger role.");
 +        zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
 +        zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
  
 -          final byte[] zero = new byte[] {'0'};
 -          log.debug("Initializing recovery area.");
 -          zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
 +        final byte[] zero = new byte[] {'0'};
 +        log.debug("Initializing recovery area.");
 +        zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
  
 -          for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
 -            log.debug("Prepping table " + id + " for compaction cancellations.");
 -            zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
 +        for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
 +          log.debug("Prepping table " + id + " for compaction cancellations.");
 +          zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
 +        }
 +
 +        @SuppressWarnings("deprecation")
 +        String zpath = zooRoot + Constants.ZCONFIG + "/" + Property.TSERV_WAL_SYNC_METHOD.getKey();
 +        // is the entire instance set to use flushing vs sync?
 +        boolean flushDefault = false;
 +        try {
 +          byte data[] = zoo.getData(zpath, null);
-           if (new String(data, StandardCharsets.UTF_8).endsWith("flush")) {
++          if (new String(data, UTF_8).endsWith("flush")) {
 +            flushDefault = true;
 +          }
 +        } catch (KeeperException.NoNodeException ex) {
 +          // skip
 +        }
 +        for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
 +          log.debug("Converting table " + id + " WALog setting to Durability");
 +          try {
 +            @SuppressWarnings("deprecation")
 +            String path = zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/" + Property.TABLE_WALOG_ENABLED.getKey();
 +            byte[] data = zoo.getData(path, null);
-             boolean useWAL = Boolean.parseBoolean(new String(data, StandardCharsets.UTF_8));
++            boolean useWAL = Boolean.parseBoolean(new String(data, UTF_8));
 +            zoo.recursiveDelete(path, NodeMissingPolicy.FAIL);
 +            path = zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/" + Property.TABLE_DURABILITY.getKey();
 +            if (useWAL) {
 +              if (flushDefault) {
 +                zoo.putPersistentData(path, "flush".getBytes(), NodeExistsPolicy.SKIP);
 +              } else {
 +                zoo.putPersistentData(path, "sync".getBytes(), NodeExistsPolicy.SKIP);
 +              }
 +            } else {
 +              zoo.putPersistentData(path, "none".getBytes(), NodeExistsPolicy.SKIP);
 +            }
 +          } catch (KeeperException.NoNodeException ex) {
 +            // skip it
            }
          }
  
@@@ -385,8 -342,8 +386,8 @@@
  
          // rename metadata table
          log.debug("Upgrade renaming table " + MetadataTable.OLD_NAME + " (ID: " + MetadataTable.ID + ") to " + MetadataTable.NAME);
 -        zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8),
 -            NodeExistsPolicy.OVERWRITE);
 +        zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME,
-             Tables.qualify(MetadataTable.NAME).getSecond().getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
++            Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
  
          moveRootTabletToRootTable(zoo);
  
@@@ -1087,41 -1030,6 +1088,41 @@@
      while (!clientService.isServing()) {
        UtilWaitThread.sleep(100);
      }
 +
 +    // Start the daemon to scan the replication table and make units of work
 +    replicationWorkDriver = new ReplicationDriver(this);
 +    replicationWorkDriver.start();
 +
 +    // Start the daemon to assign work to tservers to replicate to our peers
 +    try {
 +      replicationWorkAssigner = new WorkDriver(this, getConnector());
 +    } catch (AccumuloException | AccumuloSecurityException e) {
 +      log.error("Caught exception trying to initialize replication WorkDriver", e);
 +      throw new RuntimeException(e);
 +    }
 +    replicationWorkAssigner.start();
 +
 +    // Start the replication coordinator which assigns tservers to service replication requests
 +    ReplicationCoordinator.Processor<ReplicationCoordinator.Iface> replicationCoordinatorProcessor = new ReplicationCoordinator.Processor<ReplicationCoordinator.Iface>(
 +        RpcWrapper.service(new MasterReplicationCoordinator(this)));
 +    ServerAddress replAddress = TServerUtils.startServer(getConfiguration(), hostname, Property.MASTER_REPLICATION_COORDINATOR_PORT,
 +        replicationCoordinatorProcessor, "Master Replication Coordinator", "Replication Coordinator", null, Property.MASTER_REPLICATION_COORDINATOR_MINTHREADS,
 +        Property.MASTER_REPLICATION_COORDINATOR_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
 +
 +    log.info("Started replication coordinator service at " + replAddress.address);
 +
 +    // Advertise that port we used so peers don't have to be told what it is
 +    ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR,
-         replAddress.address.toString().getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
++        replAddress.address.toString().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +
 +    final SystemCredentials creds = SystemCredentials.get();
 +    try {
 +      ReplicationMetrics beanImpl = new ReplicationMetrics(this.instance.getConnector(creds.getPrincipal(), creds.getToken()));
 +      beanImpl.register();
 +    } catch (Exception e) {
 +      log.error("Error registering Replication metrics with JMX", e);
 +    }
 +
      while (clientService.isServing()) {
        UtilWaitThread.sleep(500);
      }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index ba7a52a,5e478cb..c636dbb
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.master.recovery;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.FileNotFoundException;
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Collection;
  import java.util.HashMap;
  import java.util.HashSet;
@@@ -114,10 -114,9 +115,10 @@@ public class RecoveryManager 
  
    }
  
 -  private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
 +  private void initiateSort(String sortId, String source, final String destination, AccumuloConfiguration aconf)
 +    throws KeeperException, InterruptedException, IOException {
      String work = source + "|" + destination;
-     new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY, aconf).addWork(sortId, work.getBytes(StandardCharsets.UTF_8));
 -    new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(UTF_8));
++    new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY, aconf).addWork(sortId, work.getBytes(UTF_8));
  
      synchronized (this) {
        sortsQueued.add(sortId);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
index 795fabb,0000000..8218c8a
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
@@@ -1,107 -1,0 +1,108 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.replication;
 +
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.util.Iterator;
 +import java.util.Random;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.replication.ReplicationConstants;
 +import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
 +import org.apache.accumulo.core.replication.thrift.ReplicationCoordinatorErrorCode;
 +import org.apache.accumulo.core.replication.thrift.ReplicationCoordinatorException;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.thrift.TException;
 +import org.apache.zookeeper.KeeperException;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.base.Preconditions;
 +
 +/**
 + * Choose a tserver to service a replication task
 + */
 +public class MasterReplicationCoordinator implements ReplicationCoordinator.Iface {
 +  private static final Logger log = LoggerFactory.getLogger(MasterReplicationCoordinator.class);
 +
 +  private final Master master;
 +  private final Instance inst;
 +  private final Random rand;
 +  private final ZooReader reader;
 +  private final SecurityOperation security;
 +
 +  public MasterReplicationCoordinator(Master master) {
 +    this(master, new ZooReader(master.getInstance().getZooKeepers(), master.getInstance().getZooKeepersSessionTimeOut()));
 +  }
 +
 +  protected MasterReplicationCoordinator(Master master, ZooReader reader) {
 +    this.master = master;
 +    this.rand = new Random(358923462l);
 +    this.inst = master.getInstance();
 +    this.reader = reader;
 +    this.security = SecurityOperation.getInstance(inst.getInstanceID(), false);
 +  }
 +
 +
 +  @Override
 +  public String getServicerAddress(String remoteTableId, TCredentials creds) throws ReplicationCoordinatorException, TException {
 +    try { 
 +      security.authenticateUser(SystemCredentials.get().toThrift(inst), creds);
 +    } catch (ThriftSecurityException e) {
 +      log.error("{} failed to authenticate for replication to {}", creds.getPrincipal(), remoteTableId);
 +      throw new ReplicationCoordinatorException(ReplicationCoordinatorErrorCode.CANNOT_AUTHENTICATE, "Could not authenticate " + creds.getPrincipal());
 +    }
 +
 +    Set<TServerInstance> tservers = master.onlineTabletServers();
 +    if (tservers.isEmpty()) {
 +      throw new ReplicationCoordinatorException(ReplicationCoordinatorErrorCode.NO_AVAILABLE_SERVERS, "No tservers are available for replication");
 +    }
 +
 +    TServerInstance tserver = getRandomTServer(tservers, rand.nextInt(tservers.size()));
 +    String replServiceAddr;
 +    try {
-       replServiceAddr = new String(reader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver.hostPort(), null), StandardCharsets.UTF_8);
++      replServiceAddr = new String(reader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver.hostPort(), null), UTF_8);
 +    } catch (KeeperException | InterruptedException e) {
 +      log.error("Could not fetch repliation service port for tserver", e);
 +      throw new ReplicationCoordinatorException(ReplicationCoordinatorErrorCode.SERVICE_CONFIGURATION_UNAVAILABLE,
 +          "Could not determine port for replication service running at " + tserver.hostPort());
 +    }
 +
 +    return replServiceAddr;
 +  }
 +
 +  protected TServerInstance getRandomTServer(Set<TServerInstance> tservers, int offset) {
 +    Preconditions.checkArgument(tservers.size() > offset, "Must provide an offset less than the size of the set");
 +    Iterator<TServerInstance> iter = tservers.iterator();
 +    while (offset > 0) {
 +      iter.next();
 +      offset--;
 +    }
 +
 +    return iter.next();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
index 3da225d,a674557..2bbbba3
--- a/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
@@@ -16,10 -16,9 +16,10 @@@
   */
  package org.apache.accumulo.master.state;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
  import org.apache.accumulo.core.master.thrift.MasterGoalState;
  import org.apache.accumulo.core.security.SecurityUtil;
  import org.apache.accumulo.core.zookeeper.ZooUtil;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
index b55b315,10cf224..ce25310
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.BufferedWriter;
  import java.io.FileNotFoundException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
index eb49a35,fd2441c..374fc24
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.Instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index 2673f03,fbaa70d..13ef68e
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.DataInput;
  import java.io.DataOutput;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index 5d032d1,037455c..9c397db
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedOutputStream;
  import java.io.BufferedWriter;
  import java.io.DataOutputStream;
@@@ -258,10 -259,9 +259,10 @@@ class WriteExportFiles extends MasterRe
      Map<String,String> siteConfig = conn.instanceOperations().getSiteConfiguration();
      Map<String,String> systemConfig = conn.instanceOperations().getSystemConfiguration();
      
 -    TableConfiguration tableConfig = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableID);
 +    ServerConfigurationFactory factory = new ServerConfigurationFactory(conn.getInstance());
 +    TableConfiguration tableConfig = factory.getTableConfiguration(tableID);
      
-     OutputStreamWriter osw = new OutputStreamWriter(dataOut, StandardCharsets.UTF_8);
+     OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
      
      // only put props that are different than defaults and higher level configurations
      zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 44ca898,05ec263..35067ce
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedInputStream;
  import java.io.BufferedReader;
  import java.io.BufferedWriter;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index 7965617,4f381ef..5d511ac
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.client.Instance;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 6961776,3e8704e..64947fb
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.master.tableOps;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.math.BigInteger;
- import java.nio.charset.StandardCharsets;
  import java.util.concurrent.locks.Lock;
  import java.util.concurrent.locks.ReentrantLock;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
index aeb07e2,a9058b7..53c7c6f
--- a/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
@@@ -16,7 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.master.tserverOps;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
  import org.apache.accumulo.core.Constants;
  import org.apache.accumulo.core.master.thrift.TabletServerStatus;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 6135fb0,75d1e9e..49bb56d
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.monitor;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.net.InetAddress;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index b0c20ec,78dd95e..7aefa1d
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.monitor;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collection;
  import java.util.SortedSet;
  import java.util.TreeSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index b631320,7812603..2feb804
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.monitor.servlets;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.io.PrintWriter;
  import java.io.StringWriter;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 2395335,bb7e690..3347647
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.monitor.servlets;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.io.InputStream;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Calendar;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index bf38064,4975428..6b2b31f
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.monitor.servlets;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.OutputStream;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
index b639226,3cd635e..405d6df
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.monitor.servlets;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Map.Entry;
  import java.util.SortedMap;
  import java.util.TreeMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index c1531cb,1c464d4..475cae1
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.monitor.servlets.trace;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Date;
  import java.util.Map;
  import java.util.Map.Entry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
----------------------------------------------------------------------
diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index e7e046e,6094911..4858b8a
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.tracer;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.net.InetSocketAddress;
  import java.net.ServerSocket;
  import java.nio.channels.ServerSocketChannel;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index ff0097a,4d9a6c2..f7bda49
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@@ -16,11 -16,12 +16,11 @@@
   */
  package org.apache.accumulo.tserver;
  
- import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  
+ import java.io.IOException;
 -
 +import org.apache.accumulo.core.conf.SiteConfiguration;
  import org.apache.accumulo.core.util.CachedConfiguration;
 -import org.apache.accumulo.server.conf.ServerConfiguration;
  import org.apache.accumulo.server.fs.VolumeManager;
  import org.apache.accumulo.server.fs.VolumeManagerImpl;
  import org.apache.accumulo.server.trace.TraceFileSystem;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 29ed8d7,94be0bb..54c75f8
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -16,19 -16,22 +16,19 @@@
   */
  package org.apache.accumulo.tserver;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
  
  import java.io.FileNotFoundException;
  import java.io.IOException;
 -import java.lang.management.GarbageCollectorMXBean;
  import java.lang.management.ManagementFactory;
 -import java.net.Socket;
  import java.net.UnknownHostException;
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
 -import java.security.SecureRandom;
  import java.util.ArrayList;
  import java.util.Arrays;
 +import java.util.Collection;
  import java.util.Collections;
  import java.util.Comparator;
 -import java.util.EnumMap;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Iterator;
@@@ -412,8 -1198,8 +412,8 @@@ public class TabletServer implements Ru
          List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
          long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
  
-       String tableId = new String(textent.getTable(), StandardCharsets.UTF_8);
+       String tableId = new String(textent.getTable(), UTF_8);
 -      if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
 +      if (!security.canScan(credentials, tableId, Tables.getNamespaceId(getInstance(), tableId), range, columns, ssiList, ssio, authorizations))
          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
  
        if (!security.userHasAuthorizations(credentials, authorizations))
@@@ -552,9 -1347,9 +552,9 @@@
      public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
          List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
        // find all of the tables that need to be scanned
 -      HashSet<String> tables = new HashSet<String>();
 +      final HashSet<String> tables = new HashSet<String>();
        for (TKeyExtent keyExtent : tbatch.keySet()) {
-         tables.add(new String(keyExtent.getTable(), StandardCharsets.UTF_8));
+         tables.add(new String(keyExtent.getTable(), UTF_8));
        }
  
        if (tables.size() != 1)
@@@ -950,14 -1755,14 +950,14 @@@
      }
  
      @Override
 -    public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
 +    public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation, TDurability tdurability) throws NotServingTabletException,
          ConstraintViolationException, ThriftSecurityException {
  
-       final String tableId = new String(tkeyExtent.getTable(), StandardCharsets.UTF_8);
 -      String tableId = new String(tkeyExtent.getTable(), UTF_8);
 -      if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
++      final String tableId = new String(tkeyExtent.getTable(), UTF_8);
 +      if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(getInstance(), tableId)))
          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 -      KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 -      Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
 +      final KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +      final Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
        if (tablet == null) {
          throw new NotServingTabletException(tkeyExtent);
        }
@@@ -2260,30 -3106,7 +2260,30 @@@
      return address;
    }
  
 -  ZooLock getLock() {
 +  private HostAndPort startReplicationService() throws UnknownHostException {
 +    ReplicationServicer.Iface repl = RpcWrapper.service(new ReplicationServicerHandler(HdfsZooInstance.getInstance()));
 +    ReplicationServicer.Processor<ReplicationServicer.Iface> processor = new ReplicationServicer.Processor<ReplicationServicer.Iface>(repl);
 +    AccumuloConfiguration conf = getConfiguration();
 +    Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null ? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
 +    ServerAddress sp = TServerUtils.startServer(conf, clientAddress.getHostText(), Property.REPLICATION_RECEIPT_SERVICE_PORT, processor,
 +        "ReplicationServicerHandler", "Replication Servicer", null, Property.REPLICATION_MIN_THREADS, Property.REPLICATION_THREADCHECK, maxMessageSizeProperty);
 +    this.replServer = sp.server;
 +    log.info("Started replication service on " + sp.address);
 +
 +    try {
 +      // The replication service is unique to the thrift service for a tserver, not just a host.
 +      // Advertise the host and port for replication service given the host and port for the tserver.
 +      ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(getInstance()) + ReplicationConstants.ZOO_TSERVERS + "/" + clientAddress.toString(),
-           sp.address.toString().getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
++          sp.address.toString().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    } catch (Exception e) {
 +      log.error("Could not advertise replication service port", e);
 +      throw new RuntimeException(e);
 +    }
 +
 +    return sp.address;
 +  }
 +
 +  public ZooLock getLock() {
      return tabletServerLock;
    }
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 0ea455c,7158ea8..b7d5b0e
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@@ -16,14 -16,21 +16,20 @@@
   */
  package org.apache.accumulo.tserver.log;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_FINISH;
+ import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_START;
+ import static org.apache.accumulo.tserver.logger.LogEvents.DEFINE_TABLET;
+ import static org.apache.accumulo.tserver.logger.LogEvents.MANY_MUTATIONS;
+ import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
+ 
  import java.io.DataInputStream;
  import java.io.DataOutputStream;
  import java.io.EOFException;
  import java.io.IOException;
  import java.io.OutputStream;
 -import java.lang.reflect.InvocationTargetException;
  import java.lang.reflect.Method;
  import java.nio.channels.ClosedChannelException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
index 1468fec,2c807fd..81cd593
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.tserver.logger;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.DataInput;
  import java.io.DataOutput;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index a444b45,29eefc8..df2f831
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.tserver.logger;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.DataInputStream;
  import java.io.EOFException;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
index fbae6f2,0000000..4b57813
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
@@@ -1,181 -1,0 +1,182 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.replication;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 +import java.util.Map;
 +import java.util.NoSuchElementException;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.replication.ReplicaSystem;
 +import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.protobuf.ProtobufUtil;
 +import org.apache.accumulo.core.replication.ReplicaSystemHelper;
 +import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
 +import org.apache.accumulo.core.replication.ReplicationTarget;
 +import org.apache.accumulo.core.replication.StatusUtil;
 +import org.apache.accumulo.core.replication.proto.Replication.Status;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.server.conf.ServerConfigurationFactory;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.replication.DistributedWorkQueueWorkAssignerHelper;
 +import org.apache.accumulo.server.replication.ReplicationTable;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
 +import org.apache.hadoop.fs.Path;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.collect.Iterables;
 +import com.google.protobuf.InvalidProtocolBufferException;
 +
 +/**
 + * Transmit the given data to a peer
 + */
 +public class ReplicationProcessor implements Processor {
 +  private static final Logger log = LoggerFactory.getLogger(ReplicationProcessor.class);
 +
 +  private final Instance inst;
 +  private final AccumuloConfiguration conf;
 +  private final VolumeManager fs;
 +  private final Credentials creds;
 +  private final ReplicaSystemHelper helper;
 +
 +  public ReplicationProcessor(Instance inst, AccumuloConfiguration conf, VolumeManager fs, Credentials creds) {
 +    this.inst = inst;
 +    this.conf = conf;
 +    this.fs = fs;
 +    this.creds = creds;
 +    this.helper = new ReplicaSystemHelper(inst, creds);
 +  }
 +
 +  @Override
 +  public ReplicationProcessor newProcessor() {
 +    return new ReplicationProcessor(inst, new ServerConfigurationFactory(inst).getConfiguration(), fs, creds);
 +  }
 +
 +  @Override
 +  public void process(String workID, byte[] data) {
 +    ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID).getValue();
-     String file = new String(data, StandardCharsets.UTF_8);
++    String file = new String(data, UTF_8);
 +
 +    log.debug("Received replication work for {} to {}", file, target);
 +
 +    ReplicaSystem replica;
 +    try {
 +      replica = getReplicaSystem(target);
 +    } catch (Exception e) {
 +      log.error("Could not instantiate ReplicaSystem for {}, waiting before returning the work", target, e);
 +      try {
 +        // TODO configurable
 +        Thread.sleep(5000);
 +      } catch (InterruptedException ie) {
 +        Thread.currentThread().interrupt();
 +      }
 +
 +      return;
 +    }
 +
 +    Status status;
 +    try {
 +      status = getStatus(file, target);
 +    } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
 +      log.error("Could not look for replication record", e);
 +      throw new IllegalStateException("Could not look for replication record", e);
 +    } catch (InvalidProtocolBufferException e) {
 +      log.error("Could not deserialize Status from Work section for {} and ", file, target);
 +      throw new RuntimeException("Could not parse Status for work record", e);
 +    } catch (NoSuchElementException e) {
 +      log.error("Assigned work for {} to {} but could not find work record", file, target);
 +      return;
 +    }
 +
 +    log.debug("Current status for {} replicating to {}: {}", file, target, ProtobufUtil.toString(status));
 +
 +    // We don't need to do anything (shouldn't have gotten this work record in the first place)
 +    if (!StatusUtil.isWorkRequired(status)) {
 +      log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
 +      return;
 +    }
 +
 +    // Sanity check that nothing bad happened and our replication source still exists
 +    Path filePath = new Path(file);
 +    try {
 +      if (!doesFileExist(filePath, target)) {
 +        return;
 +      }
 +    } catch (IOException e) {
 +      log.error("Could not determine if file exists {}", filePath, e);
 +      throw new RuntimeException(e);
 +    }
 +
 +    log.debug("Replicating {} to {} using {}", filePath, target, replica.getClass().getName());
 +
 +    replica.replicate(filePath, status, target, getHelper());
 +  }
 +
 +  protected ReplicaSystemHelper getHelper() {
 +    return helper;
 +  }
 +
 +  protected ReplicaSystem getReplicaSystem(ReplicationTarget target) {
 +    // Find the configured replication peer so we know how to replicate to it
 +    // Classname,Configuration
 +    String peerType = getPeerType(target.getPeerName());
 +
 +    // Get the peer that we're replicating to
 +    return ReplicaSystemFactory.get(peerType);
 +  }
 +
 +  protected String getPeerType(String peerName) {
 +    // Find the configured replication peer so we know how to replicate to it
 +    Map<String,String> configuredPeers = conf.getAllPropertiesWithPrefix(Property.REPLICATION_PEERS);
 +    String peerType = configuredPeers.get(Property.REPLICATION_PEERS.getKey() + peerName);
 +    if (null == peerType) {
 +      String msg = "Cannot process replication for unknown peer: " + peerName;
 +      log.warn(msg);
 +      throw new IllegalArgumentException(msg);
 +    }
 +
 +    return peerType;
 +  }
 +
 +  protected boolean doesFileExist(Path filePath, ReplicationTarget target) throws IOException {
 +    if (!fs.exists(filePath)) {
 +      log.warn("Received work request for {} and {}, but the file doesn't exist", filePath, target);
 +      return false;
 +    }
 +
 +    return true;
 +  }
 +
 +  protected Status getStatus(String file, ReplicationTarget target) throws TableNotFoundException, AccumuloException, AccumuloSecurityException,
 +      InvalidProtocolBufferException {
 +    Scanner s = ReplicationTable.getScanner(inst.getConnector(creds.getPrincipal(), creds.getToken()));
 +    s.setRange(Range.exact(file));
 +    s.fetchColumn(WorkSection.NAME, target.toText());
 +
 +    return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get());
 +  }
 +}


[46/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index d82827e,0000000..95c4f07
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1,2675 -1,0 +1,2676 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver.tablet;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.DataInputStream;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.EnumSet;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.PriorityQueue;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicLong;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.concurrent.locks.ReentrantLock;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Durability;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.impl.DurabilityImpl;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Violations;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Column;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.MapFileInfo;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVIterator;
 +import org.apache.accumulo.core.iterators.IterationInterruptedException;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
 +import org.apache.accumulo.core.master.thrift.TabletLoadState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.protobuf.ProtobufUtil;
 +import org.apache.accumulo.core.replication.ReplicationConfigurationUtil;
 +import org.apache.accumulo.core.replication.StatusUtil;
 +import org.apache.accumulo.core.replication.proto.Replication.Status;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 +import org.apache.accumulo.core.util.LocalityGroupUtil;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.tableOps.CompactionIterators;
 +import org.apache.accumulo.server.problems.ProblemReport;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.problems.ProblemType;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.FileUtil;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.ReplicationTableUtil;
 +import org.apache.accumulo.server.util.TabletOperations;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.instrument.Trace;
 +import org.apache.accumulo.tserver.InMemoryMap;
 +import org.apache.accumulo.tserver.MinorCompactionReason;
 +import org.apache.accumulo.tserver.TConstraintViolationException;
 +import org.apache.accumulo.tserver.TLevel;
 +import org.apache.accumulo.tserver.TabletServer;
 +import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
 +import org.apache.accumulo.tserver.TabletStatsKeeper;
 +import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
 +import org.apache.accumulo.tserver.TooManyFilesException;
 +import org.apache.accumulo.tserver.TservConstraintEnv;
 +import org.apache.accumulo.tserver.compaction.CompactionPlan;
 +import org.apache.accumulo.tserver.compaction.CompactionStrategy;
 +import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
 +import org.apache.accumulo.tserver.compaction.WriteParameters;
 +import org.apache.accumulo.tserver.constraints.ConstraintChecker;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.MutationReceiver;
 +import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
 +import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
 +import org.apache.accumulo.tserver.tablet.Compactor.CompactionCanceledException;
 +import org.apache.accumulo.tserver.tablet.Compactor.CompactionEnv;
 +import org.apache.commons.codec.DecoderException;
 +import org.apache.commons.codec.binary.Hex;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +import com.google.common.annotations.VisibleForTesting;
 +
 +/**
 + *
 + * Provide access to a single row range in a living TabletServer.
 + *
 + */
 +public class Tablet implements TabletCommitter {
 +  static private final Logger log = Logger.getLogger(Tablet.class);
 +  static private final List<LogEntry> NO_LOG_ENTRIES = Collections.emptyList();
 +
 +  private final TabletServer tabletServer;
 +  private final KeyExtent extent;
 +  private final TabletResourceManager tabletResources;
 +  private final DatafileManager datafileManager;
 +  private final TableConfiguration tableConfiguration;
 +  private final String tabletDirectory;
 +  private final Path location; // absolute path of this tablets dir
 +
 +  private final TabletMemory tabletMemory;
 +
 +  private final TabletTime tabletTime;
 +  private final Object timeLock = new Object();
 +  private long persistedTime;
 +
 +  private TServerInstance lastLocation;
 +  private volatile boolean tableDirChecked = false;
 +
 +  private final AtomicLong dataSourceDeletions = new AtomicLong(0);
 +
 +  public long getDataSourceDeletions() {
 +    return dataSourceDeletions.get();
 +  }
 +
 +  private final Set<ScanDataSource> activeScans = new HashSet<ScanDataSource>();
 +
 +  private static enum CloseState {
 +    OPEN, CLOSING, CLOSED, COMPLETE
 +  }
 +
 +  private volatile CloseState closeState = CloseState.OPEN;
 +
 +  private boolean updatingFlushID = false;
 +
 +  private long lastFlushID = -1;
 +  private long lastCompactID = -1;
 +
 +  private static class CompactionWaitInfo {
 +    long flushID = -1;
 +    long compactionID = -1;
 +  }
 +
 +  // stores info about user initiated major compaction that is waiting on a minor compaction to finish
 +  private CompactionWaitInfo compactionWaitInfo = new CompactionWaitInfo();
 +
 +  static enum CompactionState {
 +    WAITING_TO_START, IN_PROGRESS
 +  };
 +
 +  private volatile CompactionState minorCompactionState = null;
 +  private volatile CompactionState majorCompactionState = null;
 +
 +  private final Set<MajorCompactionReason> majorCompactionQueued = Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
 +
 +  private final AtomicReference<ConstraintChecker> constraintChecker = new AtomicReference<ConstraintChecker>();
 +
 +  private int writesInProgress = 0;
 +
 +  private final TabletStatsKeeper timer = new TabletStatsKeeper();
 +
 +  private final Rate queryRate = new Rate(0.2);
 +  private long queryCount = 0;
 +
 +  private final Rate queryByteRate = new Rate(0.2);
 +  private long queryBytes = 0;
 +
 +  private final Rate ingestRate = new Rate(0.2);
 +  private long ingestCount = 0;
 +
 +  private final Rate ingestByteRate = new Rate(0.2);
 +  private long ingestBytes = 0;
 +
 +  private byte[] defaultSecurityLabel = new byte[0];
 +
 +  private long lastMinorCompactionFinishTime = 0;
 +  private long lastMapFileImportTime = 0;
 +
 +  private volatile long numEntries = 0;
 +  private volatile long numEntriesInMemory = 0;
 +
 +  private final Rate scannedRate = new Rate(0.2);
 +  private final AtomicLong scannedCount = new AtomicLong(0);
 +
 +  private final ConfigurationObserver configObserver;
 +
 +  private final int logId;
 +
 +  @Override
 +  public int getLogId() {
 +    return logId;
 +  }
 +
 +  public static class LookupResult {
 +    public List<Range> unfinishedRanges = new ArrayList<Range>();
 +    public long bytesAdded = 0;
 +    public long dataSize = 0;
 +    public boolean closed = false;
 +  }
 +
 +  FileRef getNextMapFilename(String prefix) throws IOException {
 +    String extension = FileOperations.getNewFileExtension(tableConfiguration);
 +    checkTabletDir();
 +    return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension);
 +  }
 +
 +  private void checkTabletDir() throws IOException {
 +    if (!tableDirChecked) {
 +      FileStatus[] files = null;
 +      try {
 +        files = getTabletServer().getFileSystem().listStatus(location);
 +      } catch (FileNotFoundException ex) {
 +        // ignored
 +      }
 +
 +      if (files == null) {
 +        if (location.getName().startsWith(Constants.CLONE_PREFIX))
 +          log.debug("Tablet " + extent + " had no dir, creating " + location); // its a clone dir...
 +        else
 +          log.warn("Tablet " + extent + " had no dir, creating " + location);
 +
 +        getTabletServer().getFileSystem().mkdirs(location);
 +      }
 +      tableDirChecked = true;
 +    }
 +  }
 +
 +  /**
 +   * Only visible for testing
 +   */
 +  @VisibleForTesting
 +  protected Tablet(TabletTime tabletTime, String tabletDirectory, int logId, Path location, DatafileManager datafileManager, TabletServer tabletServer,
 +      TabletResourceManager tabletResources, TabletMemory tabletMemory, TableConfiguration tableConfiguration, KeyExtent extent,
 +      ConfigurationObserver configObserver) {
 +    this.tabletTime = tabletTime;
 +    this.tabletDirectory = tabletDirectory;
 +    this.logId = logId;
 +    this.location = location;
 +    this.datafileManager = datafileManager;
 +    this.tabletServer = tabletServer;
 +    this.tabletResources = tabletResources;
 +    this.tabletMemory = tabletMemory;
 +    this.tableConfiguration = tableConfiguration;
 +    this.extent = extent;
 +    this.configObserver = configObserver;
 +  }
 +
 +  public Tablet(TabletServer tabletServer, KeyExtent extent, TabletResourceManager trm, SplitInfo info) throws IOException {
 +    this(tabletServer, new Text(info.getDir()), extent, trm, info.getDatafiles(), info.getTime(), info.getInitFlushID(), info.getInitCompactID(), info
 +        .getLastLocation());
 +    splitCreationTime = System.currentTimeMillis();
 +  }
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<FileRef,DataFileValue> datafiles,
 +      String time, long initFlushID, long initCompactID, TServerInstance lastLocation) throws IOException {
 +    this(tabletServer, extent, location, trm, NO_LOG_ENTRIES, datafiles, time, lastLocation, new HashSet<FileRef>(), initFlushID, initCompactID);
 +  }
 +
 +  private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    SortedMap<Key,Value> entries;
 +
 +    if (extent.isRootTablet()) {
 +      return null;
 +    } else {
 +      entries = new TreeMap<Key,Value>();
 +      Text rowName = extent.getMetadataEntry();
 +      for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +        if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
 +          entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
 +        }
 +      }
 +    }
 +
 +    if (entries.size() == 1)
 +      return entries.values().iterator().next().toString();
 +    return null;
 +  }
 +
 +  private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +
 +    TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
 +
 +    if (extent.isRootTablet()) { // the meta0 tablet
 +      Path location = new Path(MetadataTableUtil.getRootTabletDir());
 +
 +      // cleanUpFiles() has special handling for delete. files
 +      FileStatus[] files = fs.listStatus(location);
 +      Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
 +      for (String good : goodPaths) {
 +        Path path = new Path(good);
 +        String filename = path.getName();
 +        FileRef ref = new FileRef(location.toString() + "/" + filename, path);
 +        DataFileValue dfv = new DataFileValue(0, 0);
 +        datafiles.put(ref, dfv);
 +      }
 +    } else {
 +
 +      Text rowName = extent.getMetadataEntry();
 +
 +      String tableId = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
 +      ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), tableId, Authorizations.EMPTY);
 +
 +      // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
 +      // reduced batch size to improve performance
 +      // changed here after endKeys were implemented from 10 to 1000
 +      mdScanner.setBatchSize(1000);
 +
 +      // leave these in, again, now using endKey for safety
 +      mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +
 +      mdScanner.setRange(new Range(rowName));
 +
 +      for (Entry<Key,Value> entry : mdScanner) {
 +        if (entry.getKey().compareRow(rowName) != 0) {
 +          break;
 +        }
 +
 +        FileRef ref = new FileRef(fs, entry.getKey());
 +        datafiles.put(ref, new DataFileValue(entry.getValue().get()));
 +      }
 +    }
 +    return datafiles;
 +  }
 +
 +  private static List<LogEntry> lookupLogEntries(KeyExtent ke, SortedMap<Key,Value> tabletsKeyValues) {
 +    List<LogEntry> logEntries = new ArrayList<LogEntry>();
 +
 +    if (ke.isMeta()) {
 +      try {
 +        logEntries = MetadataTableUtil.getLogEntries(SystemCredentials.get(), ke);
 +      } catch (Exception ex) {
 +        throw new RuntimeException("Unable to read tablet log entries", ex);
 +      }
 +    } else {
 +      log.debug("Looking at metadata " + tabletsKeyValues);
 +      Text row = ke.getMetadataEntry();
 +      for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +        Key key = entry.getKey();
 +        if (key.getRow().equals(row)) {
 +          if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
 +            logEntries.add(LogEntry.fromKeyValue(key, entry.getValue()));
 +          }
 +        }
 +      }
 +    }
 +
 +    log.debug("got " + logEntries + " for logs for " + ke);
 +    return logEntries;
 +  }
 +
 +  private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
 +    HashSet<FileRef> scanFiles = new HashSet<FileRef>();
 +
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
 +        scanFiles.add(new FileRef(fs, key));
 +      }
 +    }
 +
 +    return scanFiles;
 +  }
 +
 +  private static long lookupFlushID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +        return Long.parseLong(entry.getValue().toString());
 +    }
 +
 +    return -1;
 +  }
 +
 +  private static long lookupCompactID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +        return Long.parseLong(entry.getValue().toString());
 +    }
 +
 +    return -1;
 +  }
 +
 +  private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
 +        return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
 +      }
 +    }
 +    return null;
 +  }
 +
 +  public Tablet(TabletServer tabletServer, KeyExtent extent, Text location, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
 +      throws IOException {
 +    this(tabletServer, extent, location, trm, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getConfiguration(),
 +        tabletServer.getFileSystem(), extent, tabletsKeyValues), lookupTime(tabletServer.getConfiguration(), extent, tabletsKeyValues), lookupLastServer(
 +        extent, tabletsKeyValues), lookupScanFiles(extent, tabletsKeyValues, tabletServer.getFileSystem()), lookupFlushID(extent, tabletsKeyValues),
 +        lookupCompactID(extent, tabletsKeyValues));
 +  }
 +
 +  /**
 +   * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
 +   */
 +  private Tablet(final TabletServer tabletServer, final KeyExtent extent, final Text location, final TabletResourceManager trm,
 +      final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time, final TServerInstance lastLocation,
 +      Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
 +
 +    TableConfiguration tblConf = tabletServer.getTableConfiguration(extent);
 +    if (null == tblConf) {
 +      Tables.clearCache(tabletServer.getInstance());
 +      tblConf = tabletServer.getTableConfiguration(extent);
 +      if (null == tblConf) {
 +        // Not guaranteed to be non-null, but should be. A failed load will be re-assigned though..
 +        log.warn("Could not get table configuration for " + extent.getTableId().toString());
 +      }
 +    }
 +
 +    this.tableConfiguration = tblConf;
 +
 +    TabletFiles tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getLock(), tabletServer.getFileSystem(), extent, new TabletFiles(location.toString(),
 +        rawLogEntries, rawDatafiles), ReplicationConfigurationUtil.isEnabled(extent, this.tableConfiguration));
 +
 +    Path locationPath;
 +
 +    if (tabletPaths.dir.contains(":")) {
 +      locationPath = new Path(tabletPaths.dir.toString());
 +    } else {
 +      locationPath = tabletServer.getFileSystem().getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString());
 +    }
 +
 +    final List<LogEntry> logEntries = tabletPaths.logEntries;
 +    final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
 +
 +    this.location = locationPath;
 +    this.lastLocation = lastLocation;
 +    this.tabletDirectory = tabletPaths.dir;
 +
 +    this.extent = extent;
 +    this.tabletResources = trm;
 +
 +    this.lastFlushID = initFlushID;
 +    this.lastCompactID = initCompactID;
 +
 +    if (extent.isRootTablet()) {
 +      long rtime = Long.MIN_VALUE;
 +      for (FileRef ref : datafiles.keySet()) {
 +        Path path = ref.path();
 +        FileSystem ns = tabletServer.getFileSystem().getVolumeByPath(path).getFileSystem();
 +        FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
 +        long maxTime = -1;
 +        try {
 +
 +          while (reader.hasTop()) {
 +            maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
 +            reader.next();
 +          }
 +
 +        } finally {
 +          reader.close();
 +        }
 +
 +        if (maxTime > rtime) {
 +          time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
 +          rtime = maxTime;
 +        }
 +      }
 +    }
 +    if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
 +      // recovery... old root tablet has no data, so time doesn't matter:
 +      time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
 +    }
 +
 +    this.tabletServer = tabletServer;
 +    this.logId = tabletServer.createLogId(extent);
 +
 +    setupDefaultSecurityLabels(extent);
 +
 +    tabletMemory = new TabletMemory(this);
 +    tabletTime = TabletTime.getInstance(time);
 +    persistedTime = tabletTime.getTime();
 +
 +    tableConfiguration.addObserver(configObserver = new ConfigurationObserver() {
 +
 +      private void reloadConstraints() {
 +        constraintChecker.set(new ConstraintChecker(tableConfiguration));
 +      }
 +
 +      @Override
 +      public void propertiesChanged() {
 +        reloadConstraints();
 +
 +        try {
 +          setupDefaultSecurityLabels(extent);
 +        } catch (Exception e) {
 +          log.error("Failed to reload default security labels for extent: " + extent.toString());
 +        }
 +      }
 +
 +      @Override
 +      public void propertyChanged(String prop) {
 +        if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
 +          reloadConstraints();
 +        else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
 +          try {
 +            log.info("Default security labels changed for extent: " + extent.toString());
 +            setupDefaultSecurityLabels(extent);
 +          } catch (Exception e) {
 +            log.error("Failed to reload default security labels for extent: " + extent.toString());
 +          }
 +        }
 +
 +      }
 +
 +      @Override
 +      public void sessionExpired() {
 +        log.debug("Session expired, no longer updating per table props...");
 +      }
 +
 +    });
 +
 +    tableConfiguration.getNamespaceConfiguration().addObserver(configObserver);
 +
 +    // Force a load of any per-table properties
 +    configObserver.propertiesChanged();
 +
 +    if (!logEntries.isEmpty()) {
 +      log.info("Starting Write-Ahead Log recovery for " + this.extent);
 +      // count[0] = entries used on tablet
 +      // count[1] = track max time from walog entries wihtout timestamps
 +      final long[] count = new long[2];
 +      final CommitSession commitSession = getTabletMemory().getCommitSession();
 +      count[1] = Long.MIN_VALUE;
 +      try {
 +        Set<String> absPaths = new HashSet<String>();
 +        for (FileRef ref : datafiles.keySet())
 +          absPaths.add(ref.path().toString());
 +
 +        tabletServer.recover(this.getTabletServer().getFileSystem(), extent, tableConfiguration, logEntries, absPaths, new MutationReceiver() {
 +          @Override
 +          public void receive(Mutation m) {
 +            // LogReader.printMutation(m);
 +            Collection<ColumnUpdate> muts = m.getUpdates();
 +            for (ColumnUpdate columnUpdate : muts) {
 +              if (!columnUpdate.hasTimestamp()) {
 +                // if it is not a user set timestamp, it must have been set
 +                // by the system
 +                count[1] = Math.max(count[1], columnUpdate.getTimestamp());
 +              }
 +            }
 +            getTabletMemory().mutate(commitSession, Collections.singletonList(m));
 +            count[0]++;
 +          }
 +        });
 +
 +        if (count[1] != Long.MIN_VALUE) {
 +          tabletTime.useMaxTimeFromWALog(count[1]);
 +        }
 +        commitSession.updateMaxCommittedTime(tabletTime.getTime());
 +
 +        if (count[0] == 0) {
 +          log.debug("No replayed mutations applied, removing unused entries for " + extent);
 +          MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
 +
 +          // No replication update to be made because the fact that this tablet didn't use any mutations
 +          // from the WAL implies nothing about use of this WAL by other tablets. Do nothing.
 +
 +          logEntries.clear();
 +        } else if (ReplicationConfigurationUtil.isEnabled(extent, tabletServer.getTableConfiguration(extent))) {
 +          // The logs are about to be re-used by this tablet, we need to record that they have data for this extent,
 +          // but that they may get more data. logEntries is not cleared which will cause the elements
 +          // in logEntries to be added to the currentLogs for this Tablet below.
 +          //
 +          // This update serves the same purpose as an update during a MinC. We know that the WAL was defined
 +          // (written when the WAL was opened) but this lets us know there are mutations written to this WAL
 +          // that could potentially be replicated. Because the Tablet is using this WAL, we can be sure that
 +          // the WAL isn't closed (WRT replication Status) and thus we're safe to update its progress.
 +          Status status = StatusUtil.openWithUnknownLength();
 +          for (LogEntry logEntry : logEntries) {
 +            log.debug("Writing updated status to metadata table for " + logEntry.logSet + " " + ProtobufUtil.toString(status));
 +            ReplicationTableUtil.updateFiles(SystemCredentials.get(), extent, logEntry.logSet, status);
 +          }
 +        }
 +
 +      } catch (Throwable t) {
 +        if (tableConfiguration.getBoolean(Property.TABLE_FAILURES_IGNORE)) {
 +          log.warn("Error recovering from log files: ", t);
 +        } else {
 +          throw new RuntimeException(t);
 +        }
 +      }
 +      // make some closed references that represent the recovered logs
 +      currentLogs = new HashSet<DfsLogger>();
 +      for (LogEntry logEntry : logEntries) {
 +        for (String log : logEntry.logSet) {
 +          currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), log, logEntry.getColumnQualifier().toString()));
 +        }
 +      }
 +
 +      log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + getTabletMemory().getNumEntries()
 +          + " entries created)");
 +    }
 +
 +    String contextName = tableConfiguration.get(Property.TABLE_CLASSPATH);
 +    if (contextName != null && !contextName.equals("")) {
 +      // initialize context classloader, instead of possibly waiting for it to initialize for a scan
 +      // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292
 +      AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
 +    }
 +
 +    // do this last after tablet is completely setup because it
 +    // could cause major compaction to start
 +    datafileManager = new DatafileManager(this, datafiles);
 +
 +    computeNumEntries();
 +
 +    getDatafileManager().removeFilesAfterScan(scanFiles);
 +
 +    // look for hints of a failure on the previous tablet server
 +    if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
 +      // look for any temp files hanging around
 +      removeOldTemporaryFiles();
 +    }
 +
 +    log.log(TLevel.TABLET_HIST, extent + " opened");
 +  }
 +
 +  private void removeOldTemporaryFiles() {
 +    // remove any temporary files created by a previous tablet server
 +    try {
 +      for (FileStatus tmp : getTabletServer().getFileSystem().globStatus(new Path(location, "*_tmp"))) {
 +        try {
 +          log.debug("Removing old temp file " + tmp.getPath());
 +          getTabletServer().getFileSystem().delete(tmp.getPath());
 +        } catch (IOException ex) {
 +          log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
 +        }
 +      }
 +    } catch (IOException ex) {
 +      log.error("Error scanning for old temp files in " + location);
 +    }
 +  }
 +
 +  private void setupDefaultSecurityLabels(KeyExtent extent) {
 +    if (extent.isMeta()) {
 +      defaultSecurityLabel = new byte[0];
 +    } else {
 +      try {
 +        ColumnVisibility cv = new ColumnVisibility(tableConfiguration.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
 +        this.defaultSecurityLabel = cv.getExpression();
 +      } catch (Exception e) {
 +        log.error(e, e);
 +        this.defaultSecurityLabel = new byte[0];
 +      }
 +    }
 +  }
 +
 +  private LookupResult lookup(SortedKeyValueIterator<Key,Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, List<KVEntry> results, long maxResultsSize)
 +      throws IOException {
 +
 +    LookupResult lookupResult = new LookupResult();
 +
 +    boolean exceededMemoryUsage = false;
 +    boolean tabletClosed = false;
 +
 +    Set<ByteSequence> cfset = null;
 +    if (columnSet.size() > 0)
 +      cfset = LocalityGroupUtil.families(columnSet);
 +
 +    for (Range range : ranges) {
 +
 +      if (exceededMemoryUsage || tabletClosed) {
 +        lookupResult.unfinishedRanges.add(range);
 +        continue;
 +      }
 +
 +      int entriesAdded = 0;
 +
 +      try {
 +        if (cfset != null)
 +          mmfi.seek(range, cfset, true);
 +        else
 +          mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
 +
 +        while (mmfi.hasTop()) {
 +          Key key = mmfi.getTopKey();
 +
 +          KVEntry kve = new KVEntry(key, mmfi.getTopValue());
 +          results.add(kve);
 +          entriesAdded++;
 +          lookupResult.bytesAdded += kve.estimateMemoryUsed();
 +          lookupResult.dataSize += kve.numBytes();
 +
 +          exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
 +
 +          if (exceededMemoryUsage) {
 +            addUnfinishedRange(lookupResult, range, key, false);
 +            break;
 +          }
 +
 +          mmfi.next();
 +        }
 +
 +      } catch (TooManyFilesException tmfe) {
 +        // treat this as a closed tablet, and let the client retry
 +        log.warn("Tablet " + getExtent() + " has too many files, batch lookup can not run");
 +        handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +        tabletClosed = true;
 +      } catch (IOException ioe) {
 +        if (shutdownInProgress()) {
 +          // assume HDFS shutdown hook caused this exception
 +          log.debug("IOException while shutdown in progress ", ioe);
 +          handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +          tabletClosed = true;
 +        } else {
 +          throw ioe;
 +        }
 +      } catch (IterationInterruptedException iie) {
 +        if (isClosed()) {
 +          handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +          tabletClosed = true;
 +        } else {
 +          throw iie;
 +        }
 +      } catch (TabletClosedException tce) {
 +        handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +        tabletClosed = true;
 +      }
 +
 +    }
 +
 +    return lookupResult;
 +  }
 +
 +  private void handleTabletClosedDuringScan(List<KVEntry> results, LookupResult lookupResult, boolean exceededMemoryUsage, Range range, int entriesAdded) {
 +    if (exceededMemoryUsage)
 +      throw new IllegalStateException("tablet should not exceed memory usage or close, not both");
 +
 +    if (entriesAdded > 0)
 +      addUnfinishedRange(lookupResult, range, results.get(results.size() - 1).getKey(), false);
 +    else
 +      lookupResult.unfinishedRanges.add(range);
 +
 +    lookupResult.closed = true;
 +  }
 +
 +  private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) {
 +    if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) {
 +      Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive());
 +      lookupResult.unfinishedRanges.add(nlur);
 +    }
 +  }
 +
 +  public LookupResult lookup(List<Range> ranges, HashSet<Column> columns, Authorizations authorizations, List<KVEntry> results, long maxResultSize,
 +      List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag) throws IOException {
 +
 +    if (ranges.size() == 0) {
 +      return new LookupResult();
 +    }
 +
 +    ranges = Range.mergeOverlapping(ranges);
 +    Collections.sort(ranges);
 +
 +    Range tabletRange = extent.toDataRange();
 +    for (Range range : ranges) {
 +      // do a test to see if this range falls within the tablet, if it does not
 +      // then clip will throw an exception
 +      tabletRange.clip(range);
 +    }
 +
 +    ScanDataSource dataSource = new ScanDataSource(this, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
 +
 +    LookupResult result = null;
 +
 +    try {
 +      SortedKeyValueIterator<Key,Value> iter = new SourceSwitchingIterator(dataSource);
 +      result = lookup(iter, ranges, columns, results, maxResultSize);
 +      return result;
 +    } catch (IOException ioe) {
 +      dataSource.close(true);
 +      throw ioe;
 +    } finally {
 +      // code in finally block because always want
 +      // to return mapfiles, even when exception is thrown
 +      dataSource.close(false);
 +
 +      synchronized (this) {
 +        queryCount += results.size();
 +        if (result != null)
 +          queryBytes += result.dataSize;
 +      }
 +    }
 +  }
 +
 +  Batch nextBatch(SortedKeyValueIterator<Key,Value> iter, Range range, int num, Set<Column> columns) throws IOException {
 +
 +    // log.info("In nextBatch..");
 +
 +    List<KVEntry> results = new ArrayList<KVEntry>();
 +    Key key = null;
 +
 +    Value value;
 +    long resultSize = 0L;
 +    long resultBytes = 0L;
 +
 +    long maxResultsSize = tableConfiguration.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
 +
 +    if (columns.size() == 0) {
 +      iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
 +    } else {
 +      iter.seek(range, LocalityGroupUtil.families(columns), true);
 +    }
 +
 +    Key continueKey = null;
 +    boolean skipContinueKey = false;
 +
 +    boolean endOfTabletReached = false;
 +    while (iter.hasTop()) {
 +
 +      value = iter.getTopValue();
 +      key = iter.getTopKey();
 +
 +      KVEntry kvEntry = new KVEntry(key, value); // copies key and value
 +      results.add(kvEntry);
 +      resultSize += kvEntry.estimateMemoryUsed();
 +      resultBytes += kvEntry.numBytes();
 +
 +      if (resultSize >= maxResultsSize || results.size() >= num) {
 +        continueKey = new Key(key);
 +        skipContinueKey = true;
 +        break;
 +      }
 +
 +      iter.next();
 +    }
 +
 +    if (iter.hasTop() == false) {
 +      endOfTabletReached = true;
 +    }
 +
 +    if (endOfTabletReached) {
 +      continueKey = null;
 +    }
 +
 +    if (endOfTabletReached && results.size() == 0)
 +      results = null;
 +
 +    return new Batch(skipContinueKey, results, continueKey, resultBytes);
 +  }
 +
 +  /**
 +   * Determine if a JVM shutdown is in progress.
 +   *
 +   */
 +  boolean shutdownInProgress() {
 +    try {
 +      Runtime.getRuntime().removeShutdownHook(new Thread(new Runnable() {
 +        @Override
 +        public void run() {}
 +      }));
 +    } catch (IllegalStateException ise) {
 +      return true;
 +    }
 +
 +    return false;
 +  }
 +
 +  public Scanner createScanner(Range range, int num, Set<Column> columns, Authorizations authorizations, List<IterInfo> ssiList,
 +      Map<String,Map<String,String>> ssio, boolean isolated, AtomicBoolean interruptFlag) {
 +    // do a test to see if this range falls within the tablet, if it does not
 +    // then clip will throw an exception
 +    extent.toDataRange().clip(range);
 +
 +    ScanOptions opts = new ScanOptions(num, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag, isolated);
 +    return new Scanner(this, range, opts);
 +  }
 +
 +  DataFileValue minorCompact(VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile, boolean hasQueueTime,
 +      long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
 +    boolean failed = false;
 +    long start = System.currentTimeMillis();
 +    timer.incrementStatusMinor();
 +
 +    long count = 0;
 +
 +    try {
 +      Span span = Trace.start("write");
 +      CompactionStats stats;
 +      try {
 +        count = memTable.getNumEntries();
 +
 +        DataFileValue dfv = null;
 +        if (mergeFile != null)
 +          dfv = getDatafileManager().getDatafileSizes().get(mergeFile);
 +
 +        MinorCompactor compactor = new MinorCompactor(this, memTable, mergeFile, dfv, tmpDatafile, mincReason, tableConfiguration);
 +        stats = compactor.call();
 +      } finally {
 +        span.stop();
 +      }
 +      span = Trace.start("bringOnline");
 +      try {
 +        getDatafileManager().bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
 +            commitSession, flushId);
 +      } finally {
 +        span.stop();
 +      }
 +      return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
 +    } catch (Exception e) {
 +      failed = true;
 +      throw new RuntimeException(e);
 +    } catch (Error e) {
 +      // Weird errors like "OutOfMemoryError" when trying to create the thread for the compaction
 +      failed = true;
 +      throw new RuntimeException(e);
 +    } finally {
 +      try {
 +        getTabletMemory().finalizeMinC();
 +      } catch (Throwable t) {
 +        log.error("Failed to free tablet memory", t);
 +      }
 +
 +      if (!failed) {
 +        lastMinorCompactionFinishTime = System.currentTimeMillis();
 +      }
 +      TabletServerMinCMetrics minCMetrics = getTabletServer().getMinCMetrics();
 +      if (minCMetrics.isEnabled())
 +        minCMetrics.add(TabletServerMinCMetrics.minc, (lastMinorCompactionFinishTime - start));
 +      if (hasQueueTime) {
 +        timer.updateTime(Operation.MINOR, queued, start, count, failed);
 +        if (minCMetrics.isEnabled())
 +          minCMetrics.add(TabletServerMinCMetrics.queue, (start - queued));
 +      } else
 +        timer.updateTime(Operation.MINOR, start, count, failed);
 +    }
 +  }
 +
 +  private synchronized MinorCompactionTask prepareForMinC(long flushId, MinorCompactionReason mincReason) {
 +    CommitSession oldCommitSession = getTabletMemory().prepareForMinC();
 +    otherLogs = currentLogs;
 +    currentLogs = new HashSet<DfsLogger>();
 +
 +    FileRef mergeFile = null;
 +    if (mincReason != MinorCompactionReason.RECOVERY) {
 +      mergeFile = getDatafileManager().reserveMergingMinorCompactionFile();
 +    }
 +
 +    return new MinorCompactionTask(this, mergeFile, oldCommitSession, flushId, mincReason);
 +
 +  }
 +
 +  public void flush(long tableFlushID) {
 +    boolean updateMetadata = false;
 +    boolean initiateMinor = false;
 +
 +    try {
 +
 +      synchronized (this) {
 +
 +        // only want one thing at a time to update flush ID to ensure that metadata table and tablet in memory state are consistent
 +        if (updatingFlushID)
 +          return;
 +
 +        if (lastFlushID >= tableFlushID)
 +          return;
 +
 +        if (isClosing() || isClosed() || getTabletMemory().memoryReservedForMinC())
 +          return;
 +
 +        if (getTabletMemory().getMemTable().getNumEntries() == 0) {
 +          lastFlushID = tableFlushID;
 +          updatingFlushID = true;
 +          updateMetadata = true;
 +        } else
 +          initiateMinor = true;
 +      }
 +
 +      if (updateMetadata) {
 +        Credentials creds = SystemCredentials.get();
 +        // if multiple threads were allowed to update this outside of a sync block, then it would be
 +        // a race condition
 +        MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, getTabletServer().getLock());
 +      } else if (initiateMinor)
 +        initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
 +
 +    } finally {
 +      if (updateMetadata) {
 +        synchronized (this) {
 +          updatingFlushID = false;
 +          this.notifyAll();
 +        }
 +      }
 +    }
 +
 +  }
 +
 +  public boolean initiateMinorCompaction(MinorCompactionReason mincReason) {
 +    if (isClosed()) {
 +      // don't bother trying to get flush id if closed... could be closed after this check but that is ok... just trying to cut down on uneeded log messages....
 +      return false;
 +    }
 +
 +    // get the flush id before the new memmap is made available for write
 +    long flushId;
 +    try {
 +      flushId = getFlushID();
 +    } catch (NoNodeException e) {
 +      log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
 +      return false;
 +    }
 +    return initiateMinorCompaction(flushId, mincReason);
 +  }
 +
 +  public boolean minorCompactNow(MinorCompactionReason mincReason) {
 +    long flushId;
 +    try {
 +      flushId = getFlushID();
 +    } catch (NoNodeException e) {
 +      log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
 +      return false;
 +    }
 +    MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
 +    if (mct == null)
 +      return false;
 +    mct.run();
 +    return true;
 +  }
 +
 +  boolean initiateMinorCompaction(long flushId, MinorCompactionReason mincReason) {
 +    MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
 +    if (mct == null)
 +      return false;
 +    getTabletResources().executeMinorCompaction(mct);
 +    return true;
 +  }
 +
 +  private MinorCompactionTask createMinorCompactionTask(long flushId, MinorCompactionReason mincReason) {
 +    MinorCompactionTask mct;
 +    long t1, t2;
 +
 +    StringBuilder logMessage = null;
 +
 +    try {
 +      synchronized (this) {
 +        t1 = System.currentTimeMillis();
 +
 +        if (isClosing() || isClosed() || majorCompactionState == CompactionState.WAITING_TO_START || getTabletMemory().memoryReservedForMinC()
 +            || getTabletMemory().getMemTable().getNumEntries() == 0 || updatingFlushID) {
 +
 +          logMessage = new StringBuilder();
 +
 +          logMessage.append(extent.toString());
 +          logMessage.append(" closeState " + closeState);
 +          logMessage.append(" majorCompactionState " + majorCompactionState);
 +          if (getTabletMemory() != null)
 +            logMessage.append(" tabletMemory.memoryReservedForMinC() " + getTabletMemory().memoryReservedForMinC());
 +          if (getTabletMemory() != null && getTabletMemory().getMemTable() != null)
 +            logMessage.append(" tabletMemory.getMemTable().getNumEntries() " + getTabletMemory().getMemTable().getNumEntries());
 +          logMessage.append(" updatingFlushID " + updatingFlushID);
 +
 +          return null;
 +        }
 +
 +        mct = prepareForMinC(flushId, mincReason);
 +        t2 = System.currentTimeMillis();
 +      }
 +    } finally {
 +      // log outside of sync block
 +      if (logMessage != null && log.isDebugEnabled())
 +        log.debug(logMessage);
 +    }
 +
 +    log.debug(String.format("MinC initiate lock %.2f secs", (t2 - t1) / 1000.0));
 +    return mct;
 +  }
 +
 +  public long getFlushID() throws NoNodeException {
 +    try {
 +      String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +          + Constants.ZTABLE_FLUSH_ID;
-       return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), StandardCharsets.UTF_8));
++      return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    } catch (NumberFormatException nfe) {
 +      throw new RuntimeException(nfe);
 +    } catch (KeeperException ke) {
 +      if (ke instanceof NoNodeException) {
 +        throw (NoNodeException) ke;
 +      } else {
 +        throw new RuntimeException(ke);
 +      }
 +    }
 +  }
 +
 +  long getCompactionCancelID() {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    try {
-       return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), StandardCharsets.UTF_8));
++      return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
 +    } catch (KeeperException e) {
 +      throw new RuntimeException(e);
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  public Pair<Long,List<IteratorSetting>> getCompactionID() throws NoNodeException {
 +    try {
 +      String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +          + Constants.ZTABLE_COMPACT_ID;
 +
-       String[] tokens = new String(ZooReaderWriter.getInstance().getData(zTablePath, null), StandardCharsets.UTF_8).split(",");
++      String[] tokens = new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8).split(",");
 +      long compactID = Long.parseLong(tokens[0]);
 +
 +      CompactionIterators iters = new CompactionIterators();
 +
 +      if (tokens.length > 1) {
 +        Hex hex = new Hex();
-         ByteArrayInputStream bais = new ByteArrayInputStream(hex.decode(tokens[1].split("=")[1].getBytes(StandardCharsets.UTF_8)));
++        ByteArrayInputStream bais = new ByteArrayInputStream(hex.decode(tokens[1].split("=")[1].getBytes(UTF_8)));
 +        DataInputStream dis = new DataInputStream(bais);
 +
 +        try {
 +          iters.readFields(dis);
 +        } catch (IOException e) {
 +          throw new RuntimeException(e);
 +        }
 +
 +        KeyExtent ke = new KeyExtent(extent.getTableId(), iters.getEndRow(), iters.getStartRow());
 +
 +        if (!ke.overlaps(extent)) {
 +          // only use iterators if compaction range overlaps
 +          iters = new CompactionIterators();
 +        }
 +      }
 +
 +      return new Pair<Long,List<IteratorSetting>>(compactID, iters.getIterators());
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    } catch (NumberFormatException nfe) {
 +      throw new RuntimeException(nfe);
 +    } catch (KeeperException ke) {
 +      if (ke instanceof NoNodeException) {
 +        throw (NoNodeException) ke;
 +      } else {
 +        throw new RuntimeException(ke);
 +      }
 +    } catch (DecoderException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  public synchronized void waitForMinC() {
 +    getTabletMemory().waitForMinC();
 +  }
 +
 +  private synchronized CommitSession finishPreparingMutations(long time) {
 +    if (writesInProgress < 0) {
 +      throw new IllegalStateException("waitingForLogs < 0 " + writesInProgress);
 +    }
 +
 +    if (isClosed() || getTabletMemory() == null) {
 +      return null;
 +    }
 +
 +    writesInProgress++;
 +    CommitSession commitSession = getTabletMemory().getCommitSession();
 +    commitSession.incrementCommitsInProgress();
 +    commitSession.updateMaxCommittedTime(time);
 +    return commitSession;
 +  }
 +
 +  public void checkConstraints() {
 +    ConstraintChecker cc = constraintChecker.get();
 +
 +    if (cc.classLoaderChanged()) {
 +      ConstraintChecker ncc = new ConstraintChecker(tableConfiguration);
 +      constraintChecker.compareAndSet(cc, ncc);
 +    }
 +  }
 +
 +  public CommitSession prepareMutationsForCommit(TservConstraintEnv cenv, List<Mutation> mutations) throws TConstraintViolationException {
 +
 +    ConstraintChecker cc = constraintChecker.get();
 +
 +    List<Mutation> violators = null;
 +    Violations violations = new Violations();
 +    cenv.setExtent(extent);
 +    for (Mutation mutation : mutations) {
 +      Violations more = cc.check(cenv, mutation);
 +      if (more != null) {
 +        violations.add(more);
 +        if (violators == null)
 +          violators = new ArrayList<Mutation>();
 +        violators.add(mutation);
 +      }
 +    }
 +
 +    long time = tabletTime.setUpdateTimes(mutations);
 +
 +    if (!violations.isEmpty()) {
 +
 +      HashSet<Mutation> violatorsSet = new HashSet<Mutation>(violators);
 +      ArrayList<Mutation> nonViolators = new ArrayList<Mutation>();
 +
 +      for (Mutation mutation : mutations) {
 +        if (!violatorsSet.contains(mutation)) {
 +          nonViolators.add(mutation);
 +        }
 +      }
 +
 +      CommitSession commitSession = null;
 +
 +      if (nonViolators.size() > 0) {
 +        // if everything is a violation, then it is expected that
 +        // code calling this will not log or commit
 +        commitSession = finishPreparingMutations(time);
 +        if (commitSession == null)
 +          return null;
 +      }
 +
 +      throw new TConstraintViolationException(violations, violators, nonViolators, commitSession);
 +    }
 +
 +    return finishPreparingMutations(time);
 +  }
 +
 +  @Override
 +  public synchronized void abortCommit(CommitSession commitSession, List<Mutation> value) {
 +    if (writesInProgress <= 0) {
 +      throw new IllegalStateException("waitingForLogs <= 0 " + writesInProgress);
 +    }
 +
 +    if (isCloseComplete() || getTabletMemory() == null) {
 +      throw new IllegalStateException("aborting commit when tablet is closed");
 +    }
 +
 +    commitSession.decrementCommitsInProgress();
 +    writesInProgress--;
 +    if (writesInProgress == 0)
 +      this.notifyAll();
 +  }
 +
 +  @Override
 +  public void commit(CommitSession commitSession, List<Mutation> mutations) {
 +
 +    int totalCount = 0;
 +    long totalBytes = 0;
 +
 +    // write the mutation to the in memory table
 +    for (Mutation mutation : mutations) {
 +      totalCount += mutation.size();
 +      totalBytes += mutation.numBytes();
 +    }
 +
 +    getTabletMemory().mutate(commitSession, mutations);
 +
 +    synchronized (this) {
 +      if (writesInProgress < 1) {
 +        throw new IllegalStateException("commiting mutations after logging, but not waiting for any log messages");
 +      }
 +
 +      if (isCloseComplete()) {
 +        throw new IllegalStateException("tablet closed with outstanding messages to the logger");
 +      }
 +
 +      getTabletMemory().updateMemoryUsageStats();
 +
 +      // decrement here in case an exception is thrown below
 +      writesInProgress--;
 +      if (writesInProgress == 0)
 +        this.notifyAll();
 +
 +      commitSession.decrementCommitsInProgress();
 +
 +      numEntries += totalCount;
 +      numEntriesInMemory += totalCount;
 +      ingestCount += totalCount;
 +      ingestBytes += totalBytes;
 +    }
 +  }
 +
 +  /**
 +   * Closes the mapfiles associated with a Tablet. If saveState is true, a minor compaction is performed.
 +   */
 +  public void close(boolean saveState) throws IOException {
 +    initiateClose(saveState, false, false);
 +    completeClose(saveState, true);
 +  }
 +
 +  void initiateClose(boolean saveState, boolean queueMinC, boolean disableWrites) {
 +
 +    if (!saveState && queueMinC) {
 +      throw new IllegalArgumentException("Not saving state on close and requesting minor compactions queue does not make sense");
 +    }
 +
 +    log.debug("initiateClose(saveState=" + saveState + " queueMinC=" + queueMinC + " disableWrites=" + disableWrites + ") " + getExtent());
 +
 +    MinorCompactionTask mct = null;
 +
 +    synchronized (this) {
 +      if (isClosed() || isClosing()) {
 +        String msg = "Tablet " + getExtent() + " already " + closeState;
 +        throw new IllegalStateException(msg);
 +      }
 +
 +      // enter the closing state, no splits, minor, or major compactions can start
 +      // should cause running major compactions to stop
 +      closeState = CloseState.CLOSING;
 +      this.notifyAll();
 +
 +      // determines if inserts and queries can still continue while minor compacting
 +      if (disableWrites) {
 +        closeState = CloseState.CLOSED;
 +      }
 +
 +      // wait for major compactions to finish, setting closing to
 +      // true should cause any running major compactions to abort
 +      while (isMajorCompactionRunning()) {
 +        try {
 +          this.wait(50);
 +        } catch (InterruptedException e) {
 +          log.error(e.toString());
 +        }
 +      }
 +
 +      while (updatingFlushID) {
 +        try {
 +          this.wait(50);
 +        } catch (InterruptedException e) {
 +          log.error(e.toString());
 +        }
 +      }
 +
 +      if (!saveState || getTabletMemory().getMemTable().getNumEntries() == 0) {
 +        return;
 +      }
 +
 +      getTabletMemory().waitForMinC();
 +
 +      try {
 +        mct = prepareForMinC(getFlushID(), MinorCompactionReason.CLOSE);
 +      } catch (NoNodeException e) {
 +        throw new RuntimeException(e);
 +      }
 +
 +      if (queueMinC) {
 +        getTabletResources().executeMinorCompaction(mct);
 +        return;
 +      }
 +
 +    }
 +
 +    // do minor compaction outside of synch block so that tablet can be read and written to while
 +    // compaction runs
 +    mct.run();
 +  }
 +
 +  private boolean closeCompleting = false;
 +
 +  synchronized void completeClose(boolean saveState, boolean completeClose) throws IOException {
 +
 +    if (!isClosing() || isCloseComplete() || closeCompleting) {
 +      throw new IllegalStateException("closeState = " + closeState);
 +    }
 +
 +    log.debug("completeClose(saveState=" + saveState + " completeClose=" + completeClose + ") " + getExtent());
 +
 +    // ensure this method is only called once, also guards against multiple
 +    // threads entering the method at the same time
 +    closeCompleting = true;
 +    closeState = CloseState.CLOSED;
 +
 +    // modify dataSourceDeletions so scans will try to switch data sources and fail because the tablet is closed
 +    dataSourceDeletions.incrementAndGet();
 +
 +    for (ScanDataSource activeScan : activeScans) {
 +      activeScan.interrupt();
 +    }
 +
 +    // wait for reads and writes to complete
 +    while (writesInProgress > 0 || activeScans.size() > 0) {
 +      try {
 +        this.wait(50);
 +      } catch (InterruptedException e) {
 +        log.error(e.toString());
 +      }
 +    }
 +
 +    getTabletMemory().waitForMinC();
 +
 +    if (saveState && getTabletMemory().getMemTable().getNumEntries() > 0) {
 +      try {
 +        prepareForMinC(getFlushID(), MinorCompactionReason.CLOSE).run();
 +      } catch (NoNodeException e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +
 +    if (saveState) {
 +      // at this point all tablet data is flushed, so do a consistency check
 +      RuntimeException err = null;
 +      for (int i = 0; i < 5; i++) {
 +        try {
 +          closeConsistencyCheck();
 +          err = null;
 +        } catch (RuntimeException t) {
 +          err = t;
 +          log.error("Consistency check fails, retrying " + t);
 +          UtilWaitThread.sleep(500);
 +        }
 +      }
 +      if (err != null) {
 +        ProblemReports.getInstance().report(new ProblemReport(extent.getTableId().toString(), ProblemType.TABLET_LOAD, this.extent.toString(), err));
 +        log.error("Tablet closed consistency check has failed for " + this.extent + " giving up and closing");
 +      }
 +    }
 +
 +    try {
 +      getTabletMemory().getMemTable().delete(0);
 +    } catch (Throwable t) {
 +      log.error("Failed to delete mem table : " + t.getMessage(), t);
 +    }
 +
 +    getTabletMemory().close();
 +
 +    // close map files
 +    getTabletResources().close();
 +
 +    log.log(TLevel.TABLET_HIST, extent + " closed");
 +
 +    tableConfiguration.getNamespaceConfiguration().removeObserver(configObserver);
 +    tableConfiguration.removeObserver(configObserver);
 +
 +    if (completeClose)
 +      closeState = CloseState.COMPLETE;
 +  }
 +
 +  private void closeConsistencyCheck() {
 +
 +    if (getTabletMemory().getMemTable().getNumEntries() != 0) {
 +      String msg = "Closed tablet " + extent + " has " + getTabletMemory().getMemTable().getNumEntries() + " entries in memory";
 +      log.error(msg);
 +      throw new RuntimeException(msg);
 +    }
 +
 +    if (getTabletMemory().memoryReservedForMinC()) {
 +      String msg = "Closed tablet " + extent + " has minor compacting memory";
 +      log.error(msg);
 +      throw new RuntimeException(msg);
 +    }
 +
 +    try {
 +      Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> fileLog = MetadataTableUtil.getFileAndLogEntries(SystemCredentials.get(), extent);
 +
 +      if (fileLog.getFirst().size() != 0) {
 +        String msg = "Closed tablet " + extent + " has walog entries in " + MetadataTable.NAME + " " + fileLog.getFirst();
 +        log.error(msg);
 +        throw new RuntimeException(msg);
 +      }
 +
 +      if (extent.isRootTablet()) {
 +        if (!fileLog.getSecond().keySet().equals(getDatafileManager().getDatafileSizes().keySet())) {
 +          String msg = "Data file in " + RootTable.NAME + " differ from in memory data " + extent + "  " + fileLog.getSecond().keySet() + "  "
 +              + getDatafileManager().getDatafileSizes().keySet();
 +          log.error(msg);
 +          throw new RuntimeException(msg);
 +        }
 +      } else {
 +        if (!fileLog.getSecond().equals(getDatafileManager().getDatafileSizes())) {
 +          String msg = "Data file in " + MetadataTable.NAME + " differ from in memory data " + extent + "  " + fileLog.getSecond() + "  "
 +              + getDatafileManager().getDatafileSizes();
 +          log.error(msg);
 +          throw new RuntimeException(msg);
 +        }
 +      }
 +
 +    } catch (Exception e) {
 +      String msg = "Failed to do close consistency check for tablet " + extent;
 +      log.error(msg, e);
 +      throw new RuntimeException(msg, e);
 +
 +    }
 +
 +    if (otherLogs.size() != 0 || currentLogs.size() != 0) {
 +      String msg = "Closed tablet " + extent + " has walog entries in memory currentLogs = " + currentLogs + "  otherLogs = " + otherLogs;
 +      log.error(msg);
 +      throw new RuntimeException(msg);
 +    }
 +
 +    // TODO check lastFlushID and lostCompactID - ACCUMULO-1290
 +  }
 +
 +  /**
 +   * Returns a Path object representing the tablet's location on the DFS.
 +   *
 +   * @return location
 +   */
 +  public Path getLocation() {
 +    return location;
 +  }
 +
 +  public synchronized boolean initiateMajorCompaction(MajorCompactionReason reason) {
 +
 +    if (isClosing() || isClosed() || !needsMajorCompaction(reason) || isMajorCompactionRunning() || majorCompactionQueued.contains(reason)) {
 +      return false;
 +    }
 +
 +    majorCompactionQueued.add(reason);
 +
 +    getTabletResources().executeMajorCompaction(getExtent(), new CompactionRunner(this, reason));
 +
 +    return false;
 +  }
 +
 +  /**
 +   * Returns true if a major compaction should be performed on the tablet.
 +   *
 +   */
 +  public boolean needsMajorCompaction(MajorCompactionReason reason) {
 +    if (isMajorCompactionRunning())
 +      return false;
 +    if (reason == MajorCompactionReason.CHOP || reason == MajorCompactionReason.USER)
 +      return true;
 +    return getTabletResources().needsMajorCompaction(getDatafileManager().getDatafileSizes(), reason);
 +  }
 +
 +  /**
 +   * Returns an int representing the total block size of the files served by this tablet.
 +   *
 +   * @return size
 +   */
 +  // this is the size of just the files
 +  public long estimateTabletSize() {
 +    long size = 0L;
 +
 +    for (DataFileValue sz : getDatafileManager().getDatafileSizes().values())
 +      size += sz.getSize();
 +
 +    return size;
 +  }
 +
 +  private boolean sawBigRow = false;
 +  private long timeOfLastMinCWhenBigFreakinRowWasSeen = 0;
 +  private long timeOfLastImportWhenBigFreakinRowWasSeen = 0;
 +  private long splitCreationTime;
 +
 +  private SplitRowSpec findSplitRow(Collection<FileRef> files) {
 +
 +    // never split the root tablet
 +    // check if we already decided that we can never split
 +    // check to see if we're big enough to split
 +
 +    long splitThreshold = tableConfiguration.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD);
 +    long maxEndRow = tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE);
 +
 +    if (extent.isRootTablet() || estimateTabletSize() <= splitThreshold) {
 +      return null;
 +    }
 +
 +    // have seen a big row before, do not bother checking unless a minor compaction or map file import has occurred.
 +    if (sawBigRow) {
 +      if (timeOfLastMinCWhenBigFreakinRowWasSeen != lastMinorCompactionFinishTime || timeOfLastImportWhenBigFreakinRowWasSeen != lastMapFileImportTime) {
 +        // a minor compaction or map file import has occurred... check again
 +        sawBigRow = false;
 +      } else {
 +        // nothing changed, do not split
 +        return null;
 +      }
 +    }
 +
 +    SortedMap<Double,Key> keys = null;
 +
 +    try {
 +      // we should make .25 below configurable
 +      keys = FileUtil.findMidPoint(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), extent.getPrevEndRow(), extent.getEndRow(),
 +          FileUtil.toPathStrings(files), .25);
 +    } catch (IOException e) {
 +      log.error("Failed to find midpoint " + e.getMessage());
 +      return null;
 +    }
 +
 +    // check to see if one row takes up most of the tablet, in which case we can not split
 +    try {
 +
 +      Text lastRow;
 +      if (extent.getEndRow() == null) {
 +        Key lastKey = (Key) FileUtil.findLastKey(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), files);
 +        lastRow = lastKey.getRow();
 +      } else {
 +        lastRow = extent.getEndRow();
 +      }
 +
 +      // We expect to get a midPoint for this set of files. If we don't get one, we have a problem.
 +      final Key mid = keys.get(.5);
 +      if (null == mid) {
 +        throw new IllegalStateException("Could not determine midpoint for files");
 +      }
 +
 +      // check to see that the midPoint is not equal to the end key
 +      if (mid.compareRow(lastRow) == 0) {
 +        if (keys.firstKey() < .5) {
 +          Key candidate = keys.get(keys.firstKey());
 +          if (candidate.getLength() > maxEndRow) {
 +            log.warn("Cannot split tablet " + extent + ", selected split point too long.  Length :  " + candidate.getLength());
 +
 +            sawBigRow = true;
 +            timeOfLastMinCWhenBigFreakinRowWasSeen = lastMinorCompactionFinishTime;
 +            timeOfLastImportWhenBigFreakinRowWasSeen = lastMapFileImportTime;
 +
 +            return null;
 +          }
 +          if (candidate.compareRow(lastRow) != 0) {
 +            // we should use this ratio in split size estimations
 +            if (log.isTraceEnabled())
 +              log.trace(String.format("Splitting at %6.2f instead of .5, row at .5 is same as end row%n", keys.firstKey()));
 +            return new SplitRowSpec(keys.firstKey(), candidate.getRow());
 +          }
 +
 +        }
 +
 +        log.warn("Cannot split tablet " + extent + " it contains a big row : " + lastRow);
 +
 +        sawBigRow = true;
 +        timeOfLastMinCWhenBigFreakinRowWasSeen = lastMinorCompactionFinishTime;
 +        timeOfLastImportWhenBigFreakinRowWasSeen = lastMapFileImportTime;
 +
 +        return null;
 +      }
 +
 +      Text text = mid.getRow();
 +      SortedMap<Double,Key> firstHalf = keys.headMap(.5);
 +      if (firstHalf.size() > 0) {
 +        Text beforeMid = firstHalf.get(firstHalf.lastKey()).getRow();
 +        Text shorter = new Text();
 +        int trunc = longestCommonLength(text, beforeMid);
 +        shorter.set(text.getBytes(), 0, Math.min(text.getLength(), trunc + 1));
 +        text = shorter;
 +      }
 +
 +      if (text.getLength() > maxEndRow) {
 +        log.warn("Cannot split tablet " + extent + ", selected split point too long.  Length :  " + text.getLength());
 +
 +        sawBigRow = true;
 +        timeOfLastMinCWhenBigFreakinRowWasSeen = lastMinorCompactionFinishTime;
 +        timeOfLastImportWhenBigFreakinRowWasSeen = lastMapFileImportTime;
 +
 +        return null;
 +      }
 +
 +      return new SplitRowSpec(.5, text);
 +    } catch (IOException e) {
 +      // don't split now, but check again later
 +      log.error("Failed to find lastkey " + e.getMessage());
 +      return null;
 +    }
 +
 +  }
 +
 +  private static int longestCommonLength(Text text, Text beforeMid) {
 +    int common = 0;
 +    while (common < text.getLength() && common < beforeMid.getLength() && text.getBytes()[common] == beforeMid.getBytes()[common]) {
 +      common++;
 +    }
 +    return common;
 +  }
 +
 +  private Map<FileRef,Pair<Key,Key>> getFirstAndLastKeys(SortedMap<FileRef,DataFileValue> allFiles) throws IOException {
 +    Map<FileRef,Pair<Key,Key>> result = new HashMap<FileRef,Pair<Key,Key>>();
 +    FileOperations fileFactory = FileOperations.getInstance();
 +    VolumeManager fs = getTabletServer().getFileSystem();
 +    for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) {
 +      FileRef file = entry.getKey();
 +      FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
 +      FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration());
 +      try {
 +        Key first = openReader.getFirstKey();
 +        Key last = openReader.getLastKey();
 +        result.put(file, new Pair<Key,Key>(first, last));
 +      } finally {
 +        openReader.close();
 +      }
 +    }
 +    return result;
 +  }
 +
 +  List<FileRef> findChopFiles(KeyExtent extent, Map<FileRef,Pair<Key,Key>> firstAndLastKeys, Collection<FileRef> allFiles) throws IOException {
 +    List<FileRef> result = new ArrayList<FileRef>();
 +    if (firstAndLastKeys == null) {
 +      result.addAll(allFiles);
 +      return result;
 +    }
 +
 +    for (FileRef file : allFiles) {
 +      Pair<Key,Key> pair = firstAndLastKeys.get(file);
 +      if (pair == null) {
 +        // file was created or imported after we obtained the first and last keys... there
 +        // are a few options here... throw an exception which will cause the compaction to
 +        // retry and also cause ugly error message that the admin has to ignore... could
 +        // go get the first and last key, but this code is called while the tablet lock
 +        // is held... or just compact the file....
 +        result.add(file);
 +      } else {
 +        Key first = pair.getFirst();
 +        Key last = pair.getSecond();
 +        // If first and last are null, it's an empty file. Add it to the compact set so it goes away.
 +        if ((first == null && last == null) || !extent.contains(first.getRow()) || !extent.contains(last.getRow())) {
 +          result.add(file);
 +        }
 +      }
 +    }
 +    return result;
 +  }
 +
 +  /**
 +   * Returns true if this tablet needs to be split
 +   *
 +   */
 +  public synchronized boolean needsSplit() {
 +    if (isClosing() || isClosed())
 +      return false;
 +    return findSplitRow(getDatafileManager().getFiles()) != null;
 +  }
 +
 +  // BEGIN PRIVATE METHODS RELATED TO MAJOR COMPACTION
 +
 +  private boolean isCompactionEnabled() {
 +    return !isClosing() && !getTabletServer().isMajorCompactionDisabled();
 +  }
 +
 +  private CompactionStats _majorCompact(MajorCompactionReason reason) throws IOException, CompactionCanceledException {
 +
 +    long t1, t2, t3;
 +
 +    // acquire file info outside of tablet lock
 +    CompactionStrategy strategy = Property.createTableInstanceFromPropertyName(tableConfiguration, Property.TABLE_COMPACTION_STRATEGY,
 +        CompactionStrategy.class, new DefaultCompactionStrategy());
 +    strategy.init(Property.getCompactionStrategyOptions(tableConfiguration));
 +
 +    Map<FileRef,Pair<Key,Key>> firstAndLastKeys = null;
 +    if (reason == MajorCompactionReason.CHOP) {
 +      firstAndLastKeys = getFirstAndLastKeys(getDatafileManager().getDatafileSizes());
 +    } else if (reason != MajorCompactionReason.USER) {
 +      MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, getTabletServer().getFileSystem(), tableConfiguration);
 +      request.setFiles(getDatafileManager().getDatafileSizes());
 +      strategy.gatherInformation(request);
 +    }
 +
 +    Map<FileRef,DataFileValue> filesToCompact;
 +
 +    int maxFilesToCompact = tableConfiguration.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
 +
 +    CompactionStats majCStats = new CompactionStats();
 +    CompactionPlan plan = null;
 +
 +    boolean propogateDeletes = false;
 +
 +    synchronized (this) {
 +      // plan all that work that needs to be done in the sync block... then do the actual work
 +      // outside the sync block
 +
 +      t1 = System.currentTimeMillis();
 +
 +      majorCompactionState = CompactionState.WAITING_TO_START;
 +
 +      getTabletMemory().waitForMinC();
 +
 +      t2 = System.currentTimeMillis();
 +
 +      majorCompactionState = CompactionState.IN_PROGRESS;
 +      notifyAll();
 +
 +      VolumeManager fs = getTabletServer().getFileSystem();
 +      if (extent.isRootTablet()) {
 +        // very important that we call this before doing major compaction,
 +        // otherwise deleted compacted files could possible be brought back
 +        // at some point if the file they were compacted to was legitimately
 +        // removed by a major compaction
 +        RootFiles.cleanupReplacement(fs, fs.listStatus(this.location), false);
 +      }
 +      SortedMap<FileRef,DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
 +      List<FileRef> inputFiles = new ArrayList<FileRef>();
 +      if (reason == MajorCompactionReason.CHOP) {
 +        // enforce rules: files with keys outside our range need to be compacted
 +        inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
 +      } else if (reason == MajorCompactionReason.USER) {
 +        inputFiles.addAll(allFiles.keySet());
 +      } else {
 +        MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, fs, tableConfiguration);
 +        request.setFiles(allFiles);
 +        plan = strategy.getCompactionPlan(request);
 +        if (plan != null) {
 +          plan.validate(allFiles.keySet());
 +          inputFiles.addAll(plan.inputFiles);
 +        }
 +      }
 +
 +      if (inputFiles.isEmpty()) {
 +        return majCStats;
 +      }
 +      // If no original files will exist at the end of the compaction, we do not have to propogate deletes
 +      Set<FileRef> droppedFiles = new HashSet<FileRef>();
 +      droppedFiles.addAll(inputFiles);
 +      if (plan != null)
 +        droppedFiles.addAll(plan.deleteFiles);
 +      propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
 +      log.debug("Major compaction plan: " + plan + " propogate deletes : " + propogateDeletes);
 +      filesToCompact = new HashMap<FileRef,DataFileValue>(allFiles);
 +      filesToCompact.keySet().retainAll(inputFiles);
 +
 +      t3 = System.currentTimeMillis();
 +
 +      getDatafileManager().reserveMajorCompactingFiles(filesToCompact.keySet());
 +    }
 +
 +    try {
 +
 +      log.debug(String.format("MajC initiate lock %.2f secs, wait %.2f secs", (t3 - t2) / 1000.0, (t2 - t1) / 1000.0));
 +
 +      Pair<Long,List<IteratorSetting>> compactionId = null;
 +      if (!propogateDeletes) {
 +        // compacting everything, so update the compaction id in metadata
 +        try {
 +          compactionId = getCompactionID();
 +        } catch (NoNodeException e) {
 +          throw new RuntimeException(e);
 +        }
 +      }
 +
 +      List<IteratorSetting> compactionIterators = new ArrayList<IteratorSetting>();
 +      if (compactionId != null) {
 +        if (reason == MajorCompactionReason.USER) {
 +          if (getCompactionCancelID() >= compactionId.getFirst()) {
 +            // compaction was canceled
 +            return majCStats;
 +          }
 +
 +          synchronized (this) {
 +            if (lastCompactID >= compactionId.getFirst())
 +              // already compacted
 +              return majCStats;
 +          }
 +        }
 +
 +        compactionIterators = compactionId.getSecond();
 +      }
 +
 +      // need to handle case where only one file is being major compacted
 +      while (filesToCompact.size() > 0) {
 +
 +        int numToCompact = maxFilesToCompact;
 +
 +        if (filesToCompact.size() > maxFilesToCompact && filesToCompact.size() < 2 * maxFilesToCompact) {
 +          // on the second to last compaction pass, compact the minimum amount of files possible
 +          numToCompact = filesToCompact.size() - maxFilesToCompact + 1;
 +        }
 +
 +        Set<FileRef> smallestFiles = removeSmallest(filesToCompact, numToCompact);
 +
 +        FileRef fileName = getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
 +        FileRef compactTmpName = new FileRef(fileName.path().toString() + "_tmp");
 +
 +        AccumuloConfiguration tableConf = createTableConfiguration(tableConfiguration, plan);
 +
 +        Span span = Trace.start("compactFiles");
 +        try {
 +
 +          CompactionEnv cenv = new CompactionEnv() {
 +            @Override
 +            public boolean isCompactionEnabled() {
 +              return Tablet.this.isCompactionEnabled();
 +            }
 +
 +            @Override
 +            public IteratorScope getIteratorScope() {
 +              return IteratorScope.majc;
 +            }
 +          };
 +
 +          HashMap<FileRef,DataFileValue> copy = new HashMap<FileRef,DataFileValue>(getDatafileManager().getDatafileSizes());
 +          if (!copy.keySet().containsAll(smallestFiles))
 +            throw new IllegalStateException("Cannot find data file values for " + smallestFiles);
 +
 +          copy.keySet().retainAll(smallestFiles);
 +
 +          log.debug("Starting MajC " + extent + " (" + reason + ") " + copy.keySet() + " --> " + compactTmpName + "  " + compactionIterators);
 +
 +          // always propagate deletes, unless last batch
 +          boolean lastBatch = filesToCompact.isEmpty();
 +          Compactor compactor = new Compactor(this, copy, null, compactTmpName, lastBatch ? propogateDeletes : true, cenv, compactionIterators,
 +              reason.ordinal(), tableConf);
 +
 +          CompactionStats mcs = compactor.call();
 +
 +          span.data("files", "" + smallestFiles.size());
 +          span.data("read", "" + mcs.getEntriesRead());
 +          span.data("written", "" + mcs.getEntriesWritten());
 +          majCStats.add(mcs);
 +
 +          if (lastBatch && plan != null && plan.deleteFiles != null) {
 +            smallestFiles.addAll(plan.deleteFiles);
 +          }
 +          getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
 +              filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null,
 +              new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
 +
 +          // when major compaction produces a file w/ zero entries, it will be deleted... do not want
 +          // to add the deleted file
 +          if (filesToCompact.size() > 0 && mcs.getEntriesWritten() > 0) {
 +            filesToCompact.put(fileName, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
 +          }
 +        } finally {
 +          span.stop();
 +        }
 +
 +      }
 +      return majCStats;
 +    } finally {
 +      synchronized (Tablet.this) {
 +        getDatafileManager().clearMajorCompactingFile();
 +      }
 +    }
 +  }
 +
 +  protected AccumuloConfiguration createTableConfiguration(TableConfiguration base, CompactionPlan plan) {
 +    if (plan == null || plan.writeParameters == null)
 +      return base;
 +    WriteParameters p = plan.writeParameters;
 +    ConfigurationCopy result = new ConfigurationCopy(base);
 +    if (p.getHdfsBlockSize() > 0)
 +      result.set(Property.TABLE_FILE_BLOCK_SIZE, "" + p.getHdfsBlockSize());
 +    if (p.getBlockSize() > 0)
 +      result.set(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE, "" + p.getBlockSize());
 +    if (p.getIndexBlockSize() > 0)
 +      result.set(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX, "" + p.getIndexBlockSize());
 +    if (p.getCompressType() != null)
 +      result.set(Property.TABLE_FILE_COMPRESSION_TYPE, p.getCompressType());
 +    if (p.getReplication() != 0)
 +      result.set(Property.TABLE_FILE_REPLICATION, "" + p.getReplication());
 +    return result;
 +  }
 +
 +  private Set<FileRef> removeSmallest(Map<FileRef,DataFileValue> filesToCompact, int maxFilesToCompact) {
 +    // ensure this method works properly when multiple files have the same size
 +
 +    PriorityQueue<Pair<FileRef,Long>> fileHeap = new PriorityQueue<Pair<FileRef,Long>>(filesToCompact.size(), new Comparator<Pair<FileRef,Long>>() {
 +      @Override
 +      public int compare(Pair<FileRef,Long> o1, Pair<FileRef,Long> o2) {
 +        if (o1.getSecond() == o2.getSecond())
 +          return o1.getFirst().compareTo(o2.getFirst());
 +        if (o1.getSecond() < o2.getSecond())
 +          return -1;
 +        return 1;
 +      }
 +    });
 +
 +    for (Iterator<Entry<FileRef,DataFileValue>> iterator = filesToCompact.entrySet().iterator(); iterator.hasNext();) {
 +      Entry<FileRef,DataFileValue> entry = iterator.next();
 +      fileHeap.add(new Pair<FileRef,Long>(entry.getKey(), entry.getValue().getSize()));
 +    }
 +
 +    Set<FileRef> smallestFiles = new HashSet<FileRef>();
 +    while (smallestFiles.size() < maxFilesToCompact && fileHeap.size() > 0) {
 +      Pair<FileRef,Long> pair = fileHeap.remove();
 +      filesToCompact.remove(pair.getFirst());
 +      smallestFiles.add(pair.getFirst());
 +    }
 +
 +    return smallestFiles;
 +  }
 +
 +  // END PRIVATE METHODS RELATED TO MAJOR COMPACTION
 +
 +  /**
 +   * Performs a major compaction on the tablet. If needsSplit() returns true, the tablet is split and a reference to the new tablet is returned.
 +   */
 +
 +  CompactionStats majorCompact(MajorCompactionReason reason, long queued) {
 +
 +    CompactionStats majCStats = null;
 +    boolean success = false;
 +    long start = System.currentTimeMillis();
 +
 +    // Always trace majC
 +    Span span = Trace.on("majorCompaction");
 +
 +    try {
 +      timer.incrementStatusMajor();
 +
 +      synchronized (this) {
 +        // check that compaction is still needed - defer to splitting
 +        majorCompactionQueued.remove(reason);
 +
 +        if (isClosing() || isClosed() || !needsMajorCompaction(reason) || isMajorCompactionRunning() || needsSplit()) {
 +          return null;
 +        }
 +
 +        majorCompactionState = CompactionState.WAITING_TO_START;
 +      }
 +
 +      try {
 +        majCStats = _majorCompact(reason);
 +        if (reason == MajorCompactionReason.CHOP) {
 +          MetadataTableUtil.chopped(getExtent(), this.getTabletServer().getLock());
 +          getTabletServer().enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.CHOPPED, extent));
 +        }
 +        success = true;
 +      } catch (CompactionCanceledException mcce) {
 +        log.debug("Major compaction canceled, extent = " + getExtent());
 +      } catch (Throwable t) {
 +        log.error("MajC Failed, extent = " + getExtent());
 +        log.error("MajC Failed, message = " + (t.getMessage() == null ? t.getClass().getName() : t.getMessage()), t);
 +      } finally {
 +        // ensure we always reset boolean, even
 +        // when an exception is thrown
 +        synchronized (this) {
 +          majorCompactionState = null;
 +          this.notifyAll();
 +        }
 +
 +        Span curr = Trace.currentTrace();
 +        curr.data("extent", "" + getExtent());
 +        if (majCStats != null) {
 +          curr.data("read", "" + majCStats.getEntriesRead());
 +          curr.data("written", "" + majCStats.getEntriesWritten());
 +        }
 +      }
 +    } finally {
 +      span.stop();
 +      long count = 0;
 +      if (majCStats != null)
 +        count = majCStats.getEntriesRead();
 +      timer.updateTime(Operation.MAJOR, queued, start, count, !success);
 +    }
 +
 +    return majCStats;
 +  }
 +
 +  /**
 +   * Returns a KeyExtent object representing this tablet's key range.
 +   *
 +   * @return extent
 +   */
 +  @Override
 +  public KeyExtent getExtent() {
 +    return extent;
 +  }
 +
 +  synchronized void computeNumEntries() {
 +    Collection<DataFileValue> vals = getDatafileManager().getDatafileSizes().values();
 +
 +    long numEntries = 0;
 +
 +    for (DataFileValue tableValue : vals) {
 +      numEntries += tableValue.getNumEntries();
 +    }
 +
 +    this.numEntriesInMemory = getTabletMemory().getNumEntries();
 +    numEntries += getTabletMemory().getNumEntries();
 +
 +    this.numEntries = numEntries;
 +  }
 +
 +  public long getNumEntries() {
 +    return numEntries;
 +  }
 +
 +  public long getNumEntriesInMemory() {
 +    return numEntriesInMemory;
 +  }
 +
 +  public synchronized boolean isClosing() {
 +    return closeState == CloseState.CLOSING;
 +  }
 +
 +  public synchronized boolean isClosed() {
 +    return closeState == CloseState.CLOSED || closeState == CloseState.COMPLETE;
 +  }
 +
 +  public synchronized boolean isCloseComplete() {
 +    return closeState == CloseState.COMPLETE;
 +  }
 +
 +  public boolean isMajorCompactionRunning() {
 +    return majorCompactionState != null;
 +  }
 +
 +  public boolean isMinorCompactionQueued() {
 +    return minorCompactionState == CompactionState.WAITING_TO_START;
 +  }
 +
 +  public boolean isMinorCompactionRunning() {
 +    return minorCompactionState == CompactionState.IN_PROGRESS;
 +  }
 +
 +  public boolean isMajorCompactionQueued() {
 +    return majorCompactionQueued.size() > 0;
 +  }
 +
 +  public TreeMap<KeyExtent,SplitInfo> split(byte[] sp) throws IOException {
 +
 +    if (sp != null && extent.getEndRow() != null && extent.getEndRow().equals(new Text(sp))) {
 +      throw new IllegalArgumentException();
 +    }
 +
 +    if (sp != null && sp.length > tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE)) {
 +      String msg = "Cannot split tablet " + extent + ", selected split point too long.  Length :  " + sp.length;
 +      log.warn(msg);
 +      throw new IOException(msg);
 +    }
 +
 +    if (extent.isRootTablet()) {
 +      String msg = "Cannot split root tablet";
 +      log.warn(msg);
 +      throw new RuntimeException(msg);
 +    }
 +
 +    try {
 +      initiateClose(true, false, false);
 +    } catch (IllegalStateException ise) {
 +      log.debug("File " + extent + " not splitting : " + ise.getMessage());
 +      return null;
 +    }
 +
 +    // obtain this info outside of synch block since it will involve opening
 +    // the map files... it is ok if the set of map files changes, because
 +    // this info is used for optimization... it is ok if map files are missing
 +    // from the set... can still query and insert into the tablet while this
 +    // map file operation is happening
 +    Map<FileRef,FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(getTabletServer().getFileSystem(),
 +        getTabletServer().getConfiguration(), getDatafileManager().getFiles());
 +
 +    synchronized (this) {
 +      // java needs tuples ...
 +      TreeMap<KeyExtent,SplitInfo> newTablets = new TreeMap<KeyExtent,SplitInfo>();
 +
 +      long t1 = System.currentTimeMillis();
 +      // choose a split point
 +      SplitRowSpec splitPoint;
 +      if (sp == null)
 +        splitPoint = findSplitRow(getDatafileManager().getFiles());
 +      else {
 +        Text tsp = new Text(sp);
 +        splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(),
 +            extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp), tsp);
 +      }
 +
 +      if (splitPoint == null || splitPoint.row == null) {
 +        log.info("had to abort split because splitRow was null");
 +        closeState = CloseState.OPEN;
 +        return null;
 +      }
 +
 +      closeState = CloseState.CLOSING;
 +      completeClose(true, false);
 +
 +      Text midRow = splitPoint.row;
 +      double splitRatio = splitPoint.splitRatio;
 +
 +      KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
 +      KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
 +
 +      String lowDirectory = TabletOperations.createTabletDirectory(getTabletServer().getFileSystem(), extent.getTableId().toString(), midRow);
 +
 +      // write new tablet information to MetadataTable
 +      SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>();
 +      SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
 +      List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
 +
 +      MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, firstAndLastRows, getDatafileManager().getDatafileSizes(), lowDatafileSizes,
 +          highDatafileSizes, highDatafilesToRemove);
 +
 +      log.debug("Files for low split " + low + "  " + lowD

<TRUNCATED>

[34/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 71b7155,0000000..78f8c7b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -1,318 -1,0 +1,320 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.tables;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.security.SecurityPermission;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.util.TablePropUtil;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.WatchedEvent;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.Watcher.Event.EventType;
 +
 +public class TableManager {
 +  private static SecurityPermission TABLE_MANAGER_PERMISSION = new SecurityPermission("tableManagerPermission");
 +
 +  private static final Logger log = Logger.getLogger(TableManager.class);
 +  private static final Set<TableObserver> observers = Collections.synchronizedSet(new HashSet<TableObserver>());
 +  private static final Map<String,TableState> tableStateCache = Collections.synchronizedMap(new HashMap<String,TableState>());
 +  private static final byte[] ZERO_BYTE = new byte[] {'0'};
 +  
 +  private static TableManager tableManager = null;
 +
 +  private final Instance instance;
 +  private ZooCache zooStateCache;
 +
 +  public static void prepareNewNamespaceState(String instanceId, String namespaceId, String namespace, NodeExistsPolicy existsPolicy) throws KeeperException,
 +      InterruptedException {
 +    log.debug("Creating ZooKeeper entries for new namespace " + namespace + " (ID: " + namespaceId + ")");
 +    String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId;
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    zoo.putPersistentData(zPath, new byte[0], existsPolicy);
-     zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(Constants.UTF8), existsPolicy);
++    zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(UTF_8), existsPolicy);
 +    zoo.putPersistentData(zPath + Constants.ZNAMESPACE_CONF, new byte[0], existsPolicy);
 +  }
 +
 +  public static void prepareNewTableState(String instanceId, String tableId, String namespaceId, String tableName, TableState state,
 +      NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
 +    // state gets created last
 +    log.debug("Creating ZooKeeper entries for new table " + tableName + " (ID: " + tableId + ") in namespace (ID: " + namespaceId + ")");
 +    Pair<String,String> qualifiedTableName = Tables.qualify(tableName);
 +    tableName = qualifiedTableName.getSecond();
 +    String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId;
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    zoo.putPersistentData(zTablePath, new byte[0], existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_CONF, new byte[0], existsPolicy);
-     zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(Constants.UTF8), existsPolicy);
-     zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(Constants.UTF8), existsPolicy);
++    zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(UTF_8), existsPolicy);
++    zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(UTF_8), existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE, existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE, existsPolicy);
 +    zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID, ZERO_BYTE, existsPolicy);
-     zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(Constants.UTF8), existsPolicy);
++    zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(UTF_8), existsPolicy);
 +  }
 +
 +  public synchronized static TableManager getInstance() {
 +    SecurityManager sm = System.getSecurityManager();
 +    if (sm != null) {
 +      sm.checkPermission(TABLE_MANAGER_PERMISSION);
 +    }
 +    if (tableManager == null)
 +      tableManager = new TableManager();
 +    return tableManager;
 +  }
 +
 +  private TableManager() {
 +    instance = HdfsZooInstance.getInstance();
 +    zooStateCache = new ZooCache(new TableStateWatcher());
 +    updateTableStateCache();
 +  }
 +
 +  public TableState getTableState(String tableId) {
 +    return tableStateCache.get(tableId);
 +  }
 +
 +  public static class IllegalTableTransitionException extends Exception {
 +    private static final long serialVersionUID = 1L;
 +
 +    final TableState oldState;
 +    final TableState newState;
 +
 +    public IllegalTableTransitionException(TableState oldState, TableState newState) {
 +      this.oldState = oldState;
 +      this.newState = newState;
 +    }
 +
 +    public TableState getOldState() {
 +      return oldState;
 +    }
 +
 +    public TableState getNewState() {
 +      return newState;
 +    }
 +
 +  }
 +
 +  public synchronized void transitionTableState(final String tableId, final TableState newState) {
 +    String statePath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE;
 +
 +    try {
-       ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(Constants.UTF8), ZooUtil.PUBLIC, new Mutator() {
++      ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(UTF_8), ZooUtil.PUBLIC, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] oldData) throws Exception {
 +          TableState oldState = TableState.UNKNOWN;
 +          if (oldData != null)
-             oldState = TableState.valueOf(new String(oldData, Constants.UTF8));
++            oldState = TableState.valueOf(new String(oldData, UTF_8));
 +          boolean transition = true;
 +          // +--------+
 +          // v |
 +          // NEW -> (ONLINE|OFFLINE)+--- DELETING
 +          switch (oldState) {
 +            case NEW:
 +              transition = (newState == TableState.OFFLINE || newState == TableState.ONLINE);
 +              break;
 +            case ONLINE: // fall-through intended
 +            case UNKNOWN:// fall through intended
 +            case OFFLINE:
 +              transition = (newState != TableState.NEW);
 +              break;
 +            case DELETING:
 +              // Can't transition to any state from DELETING
 +              transition = false;
 +              break;
 +          }
 +          if (!transition)
 +            throw new IllegalTableTransitionException(oldState, newState);
 +          log.debug("Transitioning state for table " + tableId + " from " + oldState + " to " + newState);
-           return newState.name().getBytes(Constants.UTF8);
++          return newState.name().getBytes(UTF_8);
 +        }
 +      });
 +    } catch (Exception e) {
 +      log.fatal("Failed to transition table to state " + newState);
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  private void updateTableStateCache() {
 +    synchronized (tableStateCache) {
 +      for (String tableId : zooStateCache.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES))
 +        if (zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE) != null)
 +          updateTableStateCache(tableId);
 +    }
 +  }
 +
 +  public TableState updateTableStateCache(String tableId) {
 +    synchronized (tableStateCache) {
 +      TableState tState = TableState.UNKNOWN;
 +      byte[] data = zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE);
 +      if (data != null) {
-         String sState = new String(data, Constants.UTF8);
++        String sState = new String(data, UTF_8);
 +        try {
 +          tState = TableState.valueOf(sState);
 +        } catch (IllegalArgumentException e) {
 +          log.error("Unrecognized state for table with tableId=" + tableId + ": " + sState);
 +        }
 +        tableStateCache.put(tableId, tState);
 +      }
 +      return tState;
 +    }
 +  }
 +
 +  public void addTable(String tableId, String namespaceId, String tableName, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException,
 +      NamespaceNotFoundException {
 +    prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
 +    updateTableStateCache(tableId);
 +  }
 +
 +  public void cloneTable(String srcTable, String tableId, String tableName, String namespaceId, Map<String,String> propertiesToSet,
 +      Set<String> propertiesToExclude, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
 +    prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
 +
 +    String srcTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + srcTable + Constants.ZTABLE_CONF;
 +    String newTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF;
 +    ZooReaderWriter.getInstance().recursiveCopyPersistent(srcTablePath, newTablePath, NodeExistsPolicy.OVERWRITE);
 +
 +    for (Entry<String,String> entry : propertiesToSet.entrySet())
 +      TablePropUtil.setTableProperty(tableId, entry.getKey(), entry.getValue());
 +
 +    for (String prop : propertiesToExclude)
 +      ZooReaderWriter.getInstance().recursiveDelete(
 +          Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF + "/" + prop, NodeMissingPolicy.SKIP);
 +
 +    updateTableStateCache(tableId);
 +  }
 +
 +  public void removeTable(String tableId) throws KeeperException, InterruptedException {
 +    synchronized (tableStateCache) {
 +      tableStateCache.remove(tableId);
 +      ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE,
 +          NodeMissingPolicy.SKIP);
 +      ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId, NodeMissingPolicy.SKIP);
 +    }
 +  }
 +
 +  public boolean addObserver(TableObserver to) {
 +    synchronized (observers) {
 +      synchronized (tableStateCache) {
 +        to.initialize(Collections.unmodifiableMap(tableStateCache));
 +        return observers.add(to);
 +      }
 +    }
 +  }
 +
 +  public boolean removeObserver(TableObserver to) {
 +    return observers.remove(to);
 +  }
 +
 +  private class TableStateWatcher implements Watcher {
 +    @Override
 +    public void process(WatchedEvent event) {
 +      if (log.isTraceEnabled())
 +        log.trace(event);
 +
 +      final String zPath = event.getPath();
 +      final EventType zType = event.getType();
 +
 +      String tablesPrefix = ZooUtil.getRoot(instance) + Constants.ZTABLES;
 +      String tableId = null;
 +
 +      if (zPath != null && zPath.startsWith(tablesPrefix + "/")) {
 +        String suffix = zPath.substring(tablesPrefix.length() + 1);
 +        if (suffix.contains("/")) {
 +          String[] sa = suffix.split("/", 2);
 +          if (Constants.ZTABLE_STATE.equals("/" + sa[1]))
 +            tableId = sa[0];
 +        }
 +        if (tableId == null) {
 +          log.warn("Unknown path in " + event);
 +          return;
 +        }
 +      }
 +
 +      switch (zType) {
 +        case NodeChildrenChanged:
 +          if (zPath != null && zPath.equals(tablesPrefix)) {
 +            updateTableStateCache();
 +          } else {
 +            log.warn("Unexpected path " + zPath);
 +          }
 +          break;
 +        case NodeCreated:
 +        case NodeDataChanged:
 +          // state transition
 +          TableState tState = updateTableStateCache(tableId);
 +          log.debug("State transition to " + tState + " @ " + event);
 +          synchronized (observers) {
 +            for (TableObserver to : observers)
 +              to.stateChanged(tableId, tState);
 +          }
 +          break;
 +        case NodeDeleted:
 +          if (zPath != null
 +              && tableId != null
 +              && (zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_STATE) || zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_CONF) || zPath
 +                  .equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_NAME)))
 +            tableStateCache.remove(tableId);
 +          break;
 +        case None:
 +          switch (event.getState()) {
 +            case Expired:
 +              if (log.isTraceEnabled())
 +                log.trace("Session expired " + event);
 +              synchronized (observers) {
 +                for (TableObserver to : observers)
 +                  to.sessionExpired();
 +              }
 +              break;
 +            case SyncConnected:
 +            default:
 +              if (log.isTraceEnabled())
 +                log.trace("Ignored " + event);
 +          }
 +          break;
 +        default:
 +          log.warn("Unandled " + event);
 +      }
 +    }
 +  }
 +
 +  public void removeNamespace(String namespaceId) throws KeeperException, InterruptedException {
 +    ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId, NodeMissingPolicy.SKIP);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index eabdc0d,0000000..e10c0c1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -1,79 -1,0 +1,81 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.tablets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Random;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.FastFormat;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +/**
 + * Allocates unique names for an accumulo instance. The names are unique for the lifetime of the instance.
 + * 
 + * This is useful for filenames because it makes caching easy.
 + * 
 + */
 +
 +public class UniqueNameAllocator {
 +  private long next = 0;
 +  private long maxAllocated = 0;
 +  private String nextNamePath;
 +  private Random rand;
 +  
 +  private UniqueNameAllocator() {
 +    nextNamePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZNEXT_FILE;
 +    rand = new Random();
 +  }
 +  
 +  public synchronized String getNextName() {
 +    
 +    while (next >= maxAllocated) {
 +      final int allocate = 100 + rand.nextInt(100);
 +      
 +      try {
 +        byte[] max = ZooReaderWriter.getInstance().mutate(nextNamePath, null, ZooUtil.PRIVATE, new ZooReaderWriter.Mutator() {
 +          public byte[] mutate(byte[] currentValue) throws Exception {
-             long l = Long.parseLong(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++            long l = Long.parseLong(new String(currentValue, UTF_8), Character.MAX_RADIX);
 +            l += allocate;
-             return Long.toString(l, Character.MAX_RADIX).getBytes(Constants.UTF8);
++            return Long.toString(l, Character.MAX_RADIX).getBytes(UTF_8);
 +          }
 +        });
 +        
-         maxAllocated = Long.parseLong(new String(max, Constants.UTF8), Character.MAX_RADIX);
++        maxAllocated = Long.parseLong(new String(max, UTF_8), Character.MAX_RADIX);
 +        next = maxAllocated - allocate;
 +        
 +      } catch (Exception e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +    
-     return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), Constants.UTF8);
++    return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), UTF_8);
 +  }
 +  
 +  private static UniqueNameAllocator instance = null;
 +  
 +  public static synchronized UniqueNameAllocator getInstance() {
 +    if (instance == null)
 +      instance = new UniqueNameAllocator();
 +    
 +    return instance;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index cdff21d,0000000..198b2d6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,137 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.BatchWriterOpts;
 +import org.apache.accumulo.core.client.MultiTableBatchWriter;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.cli.ClientOpts;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class AddFilesWithMissingEntries {
 +  
 +  static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
 +  
 +  public static class Opts extends ClientOpts {
 +    @Parameter(names = "-update", description = "Make changes to the " + MetadataTable.NAME + " table to include missing files")
 +    boolean update = false;
 +  }
 +  
 +  /**
 +   * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who knows what
 +   * they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
 +   */
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    BatchWriterOpts bwOpts = new BatchWriterOpts();
 +    opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
 +    
 +    final Scanner scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    scanner.setRange(MetadataSchema.TabletsSection.getRange());
 +    final Configuration conf = new Configuration();
 +    final FileSystem fs = FileSystem.get(conf);
 +    
 +    KeyExtent last = new KeyExtent();
 +    String directory = null;
 +    Set<String> knownFiles = new HashSet<String>();
 +    
 +    int count = 0;
 +    final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
 +    
 +    // collect the list of known files and the directory for each extent
 +    for (Entry<Key,Value> entry : scanner) {
 +      Key key = entry.getKey();
 +      KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
 +      // when the key extent changes
 +      if (!ke.equals(last)) {
 +        if (directory != null) {
 +          // add any files in the directory unknown to the key extent
 +          count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
 +        }
 +        directory = null;
 +        knownFiles.clear();
 +        last = ke;
 +      }
 +      if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
 +        directory = entry.getValue().toString();
 +        log.debug("Found directory " + directory + " for row " + key.getRow().toString());
 +      } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
 +        String filename = key.getColumnQualifier().toString();
 +        knownFiles.add(filename);
 +        log.debug("METADATA file found: " + filename);
 +      }
 +    }
 +    if (directory != null) {
 +      // catch the last key extent
 +      count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
 +    }
 +    log.info("There were " + count + " files that are unknown to the metadata table");
 +    writer.close();
 +  }
 +  
 +  private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update)
 +      throws Exception {
 +    int count = 0;
 +    final String tableId = ke.getTableId().toString();
 +    final Text row = ke.getMetadataEntry();
 +    log.info(row.toString());
 +    for (String dir : ServerConstants.getTablesDirs()) {
 +      final Path path = new Path(dir + "/" + tableId + directory);
 +      for (FileStatus file : fs.listStatus(path)) {
 +        if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
 +          continue;
 +        final String filename = directory + "/" + file.getPath().getName();
 +        if (!knownFiles.contains(filename)) {
 +          count++;
 +          final Mutation m = new Mutation(row);
 +          String size = Long.toString(file.getLen());
 +          String entries = "1"; // lie
 +          String value = size + "," + entries;
-           m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
++          m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(UTF_8)));
 +          if (update) {
 +            writer.getBatchWriter(MetadataTable.NAME).addMutation(m);
 +          }
 +        }
 +      }
 +    }
 +    return count;
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 2926a3f,0000000..7a1e1ba
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -1,166 -1,0 +1,167 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.List;
 +import java.util.UUID;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.volume.Volume;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.cli.ClientOpts;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.zookeeper.ZooDefs.Ids;
 +import org.apache.zookeeper.data.ACL;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class ChangeSecret {
 +  
 +  static class Opts extends ClientOpts {
 +    @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
 +    String oldPass;
 +    @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
 +    String newPass;
 +  }
 +  
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    List<String> argsList = new ArrayList<String>(args.length + 2);
 +    argsList.add("--old");
 +    argsList.add("--new");
 +    argsList.addAll(Arrays.asList(args));
 +    opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Instance inst = opts.getInstance();
 +    if (!verifyAccumuloIsDown(inst, opts.oldPass))
 +      System.exit(-1);
 +    String instanceId = rewriteZooKeeperInstance(inst, opts.oldPass, opts.newPass);
 +    updateHdfs(fs, inst, instanceId);
 +    if (opts.oldPass != null) {
 +      deleteInstance(inst, opts.oldPass);
 +    }
 +    System.out.println("New instance id is " + instanceId);
 +    System.out.println("Be sure to put your new secret in accumulo-site.xml");
 +  }
 +  
 +  interface Visitor {
 +    void visit(ZooReader zoo, String path) throws Exception;
 +  }
 +  
 +  private static void recurse(ZooReader zoo, String root, Visitor v) {
 +    try {
 +      v.visit(zoo, root);
 +      for (String child : zoo.getChildren(root)) {
 +        recurse(zoo, root + "/" + child, v);
 +      }
 +    } catch (Exception ex) {
 +      throw new RuntimeException(ex);
 +    }
 +  }
 +  
 +  private static boolean verifyAccumuloIsDown(Instance inst, String oldPassword) {
 +    ZooReader zooReader = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPassword);
 +    String root = ZooUtil.getRoot(inst);
 +    final List<String> ephemerals = new ArrayList<String>();
 +    recurse(zooReader, root, new Visitor() {
 +      public void visit(ZooReader zoo, String path) throws Exception {
 +        Stat stat = zoo.getStatus(path);
 +        if (stat.getEphemeralOwner() != 0)
 +          ephemerals.add(path);
 +      }
 +    });
 +    if (ephemerals.size() == 0) {
 +      return true;
 +    }
 +    
 +    System.err.println("The following ephemeral nodes exist, something is still running:");
 +    for (String path : ephemerals) {
 +      System.err.println(path);
 +    }
 +    return false;
 +  }
 +  
 +  private static String rewriteZooKeeperInstance(final Instance inst, String oldPass, String newPass) throws Exception {
 +    final ZooReaderWriter orig = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPass);
 +    final IZooReaderWriter new_ = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), newPass);
 +    final String newInstanceId = UUID.randomUUID().toString();
 +    String root = ZooUtil.getRoot(inst);
 +    recurse(orig, root, new Visitor() {
 +      public void visit(ZooReader zoo, String path) throws Exception {
 +        String newPath = path.replace(inst.getInstanceID(), newInstanceId);
 +        byte[] data = zoo.getData(path, null);
 +        List<ACL> acls = orig.getZooKeeper().getACL(path, new Stat());
 +        if (acls.containsAll(Ids.READ_ACL_UNSAFE)) {
 +          new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +        } else {
 +          // upgrade
 +          if (acls.containsAll(Ids.OPEN_ACL_UNSAFE)) {
 +            // make user nodes private, they contain the user's password
 +            String parts[] = path.split("/");
 +            if (parts[parts.length - 2].equals("users")) {
 +              new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +            } else {
 +              // everything else can have the readable acl
 +              new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +            }
 +          } else {
 +            new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
 +          }
 +        }
 +      }
 +    });
 +    String path = "/accumulo/instances/" + inst.getInstanceName();
 +    orig.recursiveDelete(path, NodeMissingPolicy.SKIP);
-     new_.putPersistentData(path, newInstanceId.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    new_.putPersistentData(path, newInstanceId.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    return newInstanceId;
 +  }
 +  
 +  private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
 +    // Need to recreate the instanceId on all of them to keep consistency
 +    for (Volume v : fs.getVolumes()) {
 +      final Path instanceId = ServerConstants.getInstanceIdLocation(v);
 +      if (!v.getFileSystem().delete(instanceId, true)) {
 +        throw new IOException("Could not recursively delete " + instanceId);
 +      }
 +
 +      if (!v.getFileSystem().mkdirs(instanceId)) {
 +        throw new IOException("Could not create directory " + instanceId);
 +      }
 +
 +      v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
 +    }
 +  }
 +  
 +  private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
 +    IZooReaderWriter orig = new ZooReaderWriter(origInstance.getZooKeepers(), origInstance.getZooKeepersSessionTimeOut(), oldPass);
 +    orig.recursiveDelete("/accumulo/" + origInstance.getInstanceID(), NodeMissingPolicy.SKIP);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index f2074a1,0000000..809a7c8
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -1,87 -1,0 +1,89 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class CleanZookeeper {
 +  
 +  private static final Logger log = Logger.getLogger(CleanZookeeper.class);
 +  
 +  static class Opts extends Help {
 +    @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
 +    String keepers = "localhost:2181";
 +    @Parameter(names={"--password"}, description="the system secret", password=true)
 +    String auth;
 +  }
 +  
 +  /**
 +   * @param args
 +   *          must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
 +   * @throws IOException
 +   *           error connecting to accumulo or zookeeper
 +   */
 +  public static void main(String[] args) throws IOException {
 +    Opts opts = new Opts();
 +    opts.parseArgs(CleanZookeeper.class.getName(), args);
 +    
 +    String root = Constants.ZROOT;
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +    if (opts.auth != null) {
-       zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(Constants.UTF8));
++      zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(UTF_8));
 +    }
 +    
 +    try {
 +      for (String child : zk.getChildren(root)) {
 +        if (Constants.ZINSTANCES.equals("/" + child)) {
 +          for (String instanceName : zk.getChildren(root + Constants.ZINSTANCES)) {
 +            String instanceNamePath = root + Constants.ZINSTANCES + "/" + instanceName;
 +            byte[] id = zk.getData(instanceNamePath, null);
-             if (id != null && !new String(id, Constants.UTF8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
++            if (id != null && !new String(id, UTF_8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
 +              try {
 +                zk.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
 +              } catch (KeeperException.NoAuthException ex) {
 +                log.warn("Unable to delete " + instanceNamePath);
 +              }
 +            }
 +          }
 +        } else if (!child.equals(HdfsZooInstance.getInstance().getInstanceID())) {
 +          String path = root + "/" + child;
 +          try {
 +            zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
 +          } catch (KeeperException.NoAuthException ex) {
 +            log.warn("Unable to delete " + path);
 +          }
 +        }
 +      }
 +    } catch (Exception ex) {
 +      System.out.println("Error Occurred: " + ex);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 448da86,0000000..d40830f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -1,81 -1,0 +1,83 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashSet;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class DeleteZooInstance {
 +  
 +  static class Opts extends Help {
 +    @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
 +    String instance;
 +  }
 +  
 +  static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
 +    for (int i = 0; i < 10; i++){
 +      try {
 +        zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
 +        return;
 +      } catch (KeeperException.NotEmptyException ex) {
 +        // ignored
 +      } catch (Exception ex) {
 +        throw ex;
 +      }
 +    }
 +  }
 +  
 +  /**
 +   * @param args
 +   *          : the name or UUID of the instance to be deleted
 +   */
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    opts.parseArgs(DeleteZooInstance.class.getName(), args);
 +    
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +    // try instance name:
 +    Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
 +    Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
 +    uuids.remove("instances");
 +    if (instances.contains(opts.instance)) {
 +      String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
 +      byte[] data = zk.getData(path, null);
 +      deleteRetry(zk, path);
-       deleteRetry(zk, Constants.ZROOT + "/" + new String(data, Constants.UTF8));
++      deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
 +    } else if (uuids.contains(opts.instance)) {
 +      // look for the real instance name
 +      for (String instance : instances) {
 +        String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
 +        byte[] data = zk.getData(path, null);
-         if (opts.instance.equals(new String(data, Constants.UTF8)))
++        if (opts.instance.equals(new String(data, UTF_8)))
 +          deleteRetry(zk, path);
 +      }
 +      deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 504956f,0000000..43348df
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,121 -1,0 +1,122 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.PrintStream;
 +import java.io.UnsupportedEncodingException;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.data.Stat;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class DumpZookeeper {
 +  
 +  static IZooReaderWriter zk = null;
 +  
 +  private static final Logger log = Logger.getLogger(DumpZookeeper.class);
 +  
 +  private static class Encoded {
 +    public String encoding;
 +    public String value;
 +    
 +    Encoded(String e, String v) {
 +      encoding = e;
 +      value = v;
 +    }
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = "--root", description = "the root of the znode tree to dump")
 +    String root = "/";
 +  }
 +  
 +  public static void main(String[] args) {
 +    Opts opts = new Opts();
 +    opts.parseArgs(DumpZookeeper.class.getName(), args);
 +    
 +    Logger.getRootLogger().setLevel(Level.WARN);
 +    PrintStream out = System.out;
 +    try {
 +      zk = ZooReaderWriter.getInstance();
 +      
 +      write(out, 0, "<dump root='%s'>", opts.root);
 +      for (String child : zk.getChildren(opts.root, null))
 +        if (!child.equals("zookeeper"))
 +          dump(out, opts.root, child, 1);
 +      write(out, 0, "</dump>");
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +    }
 +  }
 +  
 +  private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
 +    String path = root + "/" + child;
 +    if (root.endsWith("/"))
 +      path = root + child;
 +    Stat stat = zk.getStatus(path);
 +    if (stat == null)
 +      return;
 +    String type = "node";
 +    if (stat.getEphemeralOwner() != 0) {
 +      type = "ephemeral";
 +    }
 +    if (stat.getNumChildren() == 0) {
 +      if (stat.getDataLength() == 0) {
 +        write(out, indent, "<%s name='%s'/>", type, child);
 +      } else {
 +        Encoded value = value(path);
 +        write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
 +      }
 +    } else {
 +      if (stat.getDataLength() == 0) {
 +        write(out, indent, "<%s name='%s'>", type, child);
 +      } else {
 +        Encoded value = value(path);
 +        write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
 +      }
 +      for (String c : zk.getChildren(path, null)) {
 +        dump(out, path, c, indent + 1);
 +      }
 +      write(out, indent, "</node>");
 +    }
 +  }
 +  
 +  private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
 +    byte[] data = zk.getData(path, null);
 +    for (int i = 0; i < data.length; i++) {
 +      // does this look like simple ascii?
 +      if (data[i] < ' ' || data[i] > '~')
 +        return new Encoded("base64", Base64.encodeBase64String(data));
 +    }
-     return new Encoded(Constants.UTF8.name(), new String(data, Constants.UTF8));
++    return new Encoded(UTF_8.name(), new String(data, UTF_8));
 +  }
 +  
 +  private static void write(PrintStream out, int indent, String fmt, Object... args) {
 +    for (int i = 0; i < indent; i++)
 +      out.print(" ");
 +    out.println(String.format(fmt, args));
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f4655dc,0000000..be85b1b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -1,225 -1,0 +1,227 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Formattable;
 +import java.util.Formatter;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.UUID;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.fate.zookeeper.ZooReader;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.log4j.Logger;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class ListInstances {
 +  
 +  private static final Logger log = Logger.getLogger(ListInstances.class);
 +  
 +  private static final int NAME_WIDTH = 20;
 +  private static final int UUID_WIDTH = 37;
 +  private static final int MASTER_WIDTH = 30;
 +  
 +  private static final int ZOOKEEPER_TIMER_MILLIS = 30 * 1000;
 +  
 +  static class Opts extends Help {
 +    @Parameter(names="--print-errors", description="display errors while listing instances")
 +    boolean printErrors = false;
 +    @Parameter(names="--print-all", description="print information for all instances, not just those with names")
 +    boolean printAll = false;
 +    @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact")
 +    String keepers = null;
 +  }
 +  static Opts opts = new Opts();
 +  static int errors = 0;
 +  
 +  public static void main(String[] args) {
 +    opts.parseArgs(ListInstances.class.getName(), args);
 +    
 +    if (opts.keepers == null) {
 +      opts.keepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
 +    }
 +    
 +    String keepers = opts.keepers;
 +    boolean printAll = opts.printAll;
 +    boolean printErrors = opts.printErrors;
 +    
 +    listInstances(keepers, printAll, printErrors);
 +    
 +  }
 +
 +  static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) {
 +    errors = 0;
 +
 +    System.out.println("INFO : Using ZooKeepers " + keepers);
 +    ZooReader rdr = new ZooReader(keepers, ZOOKEEPER_TIMER_MILLIS);
 +    ZooCache cache = new ZooCache(keepers, ZOOKEEPER_TIMER_MILLIS);
 +
 +    TreeMap<String,UUID> instanceNames = getInstanceNames(rdr, printErrors);
 +    
 +    System.out.println();
 +    printHeader();
 +    
 +    for (Entry<String,UUID> entry : instanceNames.entrySet()) {
 +      printInstanceInfo(cache, entry.getKey(), entry.getValue(), printErrors);
 +    }
 +    
 +    TreeSet<UUID> instancedIds = getInstanceIDs(rdr, printErrors);
 +    instancedIds.removeAll(instanceNames.values());
 +    
 +    if (printAll) {
 +      for (UUID uuid : instancedIds) {
 +        printInstanceInfo(cache, null, uuid, printErrors);
 +      }
 +    } else if (instancedIds.size() > 0) {
 +      System.out.println();
 +      System.out.println("INFO : " + instancedIds.size() + " unamed instances were not printed, run with --print-all to see all instances");
 +    } else {
 +      System.out.println();
 +    }
 +    
 +    if (!printErrors && errors > 0) {
 +      System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info");
 +    }
 +  }
 +  
 +  private static class CharFiller implements Formattable {
 +    
 +    char c;
 +    
 +    CharFiller(char c) {
 +      this.c = c;
 +    }
 +    
 +    @Override
 +    public void formatTo(Formatter formatter, int flags, int width, int precision) {
 +      
 +      StringBuilder sb = new StringBuilder();
 +      for (int i = 0; i < width; i++)
 +        sb.append(c);
 +      formatter.format(sb.toString());
 +    }
 +    
 +  }
 +  
 +  private static void printHeader() {
 +    System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MASTER_WIDTH + "s%n", "Instance Name", "Instance ID", "Master");
 +    System.out.printf("%" + (NAME_WIDTH + 1) + "s+%" + (UUID_WIDTH + 1) + "s+%" + (MASTER_WIDTH + 1) + "s%n", new CharFiller('-'), new CharFiller('-'),
 +        new CharFiller('-'));
 +    
 +  }
 +  
 +  private static void printInstanceInfo(ZooCache cache, String instanceName, UUID iid, boolean printErrors) {
 +    String master = getMaster(cache, iid, printErrors);
 +    if (instanceName == null) {
 +      instanceName = "";
 +    }
 +    
 +    if (master == null) {
 +      master = "";
 +    }
 +    
 +    System.out.printf("%" + NAME_WIDTH + "s |%" + UUID_WIDTH + "s |%" + MASTER_WIDTH + "s%n", "\"" + instanceName + "\"", iid, master);
 +  }
 +  
 +  private static String getMaster(ZooCache cache, UUID iid, boolean printErrors) {
 +    
 +    if (iid == null) {
 +      return null;
 +    }
 +    
 +    try {
 +      String masterLocPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK;
 +      byte[] master = ZooLock.getLockData(cache, masterLocPath, null);
 +      if (master == null) {
 +        return null;
 +      }
-       return new String(master, Constants.UTF8);
++      return new String(master, UTF_8);
 +    } catch (Exception e) {
 +      handleException(e, printErrors);
 +      return null;
 +    }
 +  }
 +  
 +  private static TreeMap<String,UUID> getInstanceNames(ZooReader zk, boolean printErrors) {
 +    
 +    String instancesPath = Constants.ZROOT + Constants.ZINSTANCES;
 +    
 +    TreeMap<String,UUID> tm = new TreeMap<String,UUID>();
 +    
 +    List<String> names;
 +    
 +    try {
 +      names = zk.getChildren(instancesPath);
 +    } catch (Exception e) {
 +      handleException(e, printErrors);
 +      return tm;
 +    }
 +    
 +    for (String name : names) {
 +      String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
 +      try {
-         UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), Constants.UTF8));
++        UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), UTF_8));
 +        tm.put(name, iid);
 +      } catch (Exception e) {
 +        handleException(e, printErrors);
 +        tm.put(name, null);
 +      }
 +    }
 +    
 +    return tm;
 +  }
 +  
 +  private static TreeSet<UUID> getInstanceIDs(ZooReader zk, boolean printErrors) {
 +    TreeSet<UUID> ts = new TreeSet<UUID>();
 +    
 +    try {
 +      List<String> children = zk.getChildren(Constants.ZROOT);
 +      
 +      for (String iid : children) {
 +        if (iid.equals("instances"))
 +          continue;
 +        try {
 +          ts.add(UUID.fromString(iid));
 +        } catch (Exception e) {
 +          log.error("Exception: " + e);
 +        }
 +      }
 +    } catch (Exception e) {
 +      handleException(e, printErrors);
 +    }
 +    
 +    return ts;
 +  }
 +  
 +  private static void handleException(Exception e, boolean printErrors) {
 +    if (printErrors) {
 +      e.printStackTrace();
 +    }
 +    
 +    errors++;
 +  }
 +}


[02/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a338d5f,0000000..d5c940d
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@@ -1,553 -1,0 +1,553 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.BufferedOutputStream;
 +import java.io.File;
 +import java.io.FileOutputStream;
 +import java.net.URI;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.admin.DiskUsage;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.init.Initialize;
 +import org.apache.accumulo.server.util.Admin;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.fs.RawLocalFileSystem;
 +import org.apache.hadoop.io.Text;
 +import org.apache.zookeeper.ZooKeeper;
 +import org.junit.Assert;
 +import org.junit.Test;
 +
 +public class VolumeIT extends ConfigurableMacIT {
 +
 +  private static final Text EMPTY = new Text();
 +  private static final Value EMPTY_VALUE = new Value(new byte[] {});
 +  private File volDirBase;
 +  private Path v1, v2;
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 5 * 60;
 +  }
 +
 +  @SuppressWarnings("deprecation")
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    File baseDir = cfg.getDir();
 +    volDirBase = new File(baseDir, "volumes");
 +    File v1f = new File(volDirBase, "v1");
 +    File v2f = new File(volDirBase, "v2");
 +    v1f.mkdir();
 +    v2f.mkdir();
 +    v1 = new Path("file://" + v1f.getAbsolutePath());
 +    v2 = new Path("file://" + v2f.getAbsolutePath());
 +
 +    // Run MAC on two locations in the local file system
 +    URI v1Uri = v1.toUri();
 +    cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
 +    cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
 +    cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
 +
 +    // use raw local file system so walogs sync and flush will work
 +    hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
 +
 +    super.configure(cfg, hadoopCoreSite);
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    // create a table
 +    Connector connector = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    connector.tableOperations().create(tableName);
 +    SortedSet<Text> partitions = new TreeSet<Text>();
 +    // with some splits
 +    for (String s : "d,m,t".split(","))
 +      partitions.add(new Text(s));
 +    connector.tableOperations().addSplits(tableName, partitions);
 +    // scribble over the splits
 +    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
 +    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
 +    for (String s : rows) {
 +      Mutation m = new Mutation(new Text(s));
 +      m.put(EMPTY, EMPTY, EMPTY_VALUE);
 +      bw.addMutation(m);
 +    }
 +    bw.close();
 +    // write the data to disk, read it back
 +    connector.tableOperations().flush(tableName, null, null, true);
 +    Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
 +    int i = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      assertEquals(rows[i++], entry.getKey().getRow().toString());
 +    }
 +    // verify the new files are written to the different volumes
 +    scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    scanner.setRange(new Range("1", "1<"));
 +    scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    int fileCount = 0;
 +
 +    for (Entry<Key,Value> entry : scanner) {
 +      boolean inV1 = entry.getKey().getColumnQualifier().toString().contains(v1.toString());
 +      boolean inV2 = entry.getKey().getColumnQualifier().toString().contains(v2.toString());
 +      assertTrue(inV1 || inV2);
 +      fileCount++;
 +    }
 +    assertEquals(4, fileCount);
 +    List<DiskUsage> diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
 +    assertEquals(1, diskUsage.size());
 +    long usage = diskUsage.get(0).getUsage().longValue();
 +    System.out.println("usage " + usage);
 +    assertTrue(usage > 700 && usage < 800);
 +  }
 +
 +  private void verifyData(List<String> expected, Scanner createScanner) {
 +
 +    List<String> actual = new ArrayList<String>();
 +
 +    for (Entry<Key,Value> entry : createScanner) {
 +      Key k = entry.getKey();
 +      actual.add(k.getRow() + ":" + k.getColumnFamily() + ":" + k.getColumnQualifier() + ":" + entry.getValue());
 +    }
 +
 +    Collections.sort(expected);
 +    Collections.sort(actual);
 +
 +    Assert.assertEquals(expected, actual);
 +  }
 +
 +  @Test
 +  public void testRelativePaths() throws Exception {
 +
 +    List<String> expected = new ArrayList<String>();
 +
 +    Connector connector = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    connector.tableOperations().create(tableName, false);
 +
 +    String tableId = connector.tableOperations().tableIdMap().get(tableName);
 +
 +    SortedSet<Text> partitions = new TreeSet<Text>();
 +    // with some splits
 +    for (String s : "c,g,k,p,s,v".split(","))
 +      partitions.add(new Text(s));
 +
 +    connector.tableOperations().addSplits(tableName, partitions);
 +
 +    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    // create two files in each tablet
 +
 +    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
 +    for (String s : rows) {
 +      Mutation m = new Mutation(s);
 +      m.put("cf1", "cq1", "1");
 +      bw.addMutation(m);
 +      expected.add(s + ":cf1:cq1:1");
 +    }
 +
 +    bw.flush();
 +    connector.tableOperations().flush(tableName, null, null, true);
 +
 +    for (String s : rows) {
 +      Mutation m = new Mutation(s);
 +      m.put("cf1", "cq1", "2");
 +      bw.addMutation(m);
 +      expected.add(s + ":cf1:cq1:2");
 +    }
 +
 +    bw.close();
 +    connector.tableOperations().flush(tableName, null, null, true);
 +
 +    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
 +
 +    connector.tableOperations().offline(tableName, true);
 +
 +    connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
 +
 +    Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
 +    metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +
 +    BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +
 +    for (Entry<Key,Value> entry : metaScanner) {
 +      String cq = entry.getKey().getColumnQualifier().toString();
 +      if (cq.startsWith(v1.toString())) {
 +        Path path = new Path(cq);
 +        String relPath = "/" + path.getParent().getName() + "/" + path.getName();
 +        Mutation fileMut = new Mutation(entry.getKey().getRow());
 +        fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
 +        fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
 +        mbw.addMutation(fileMut);
 +      }
 +    }
 +
 +    mbw.close();
 +
 +    connector.tableOperations().online(tableName, true);
 +
 +    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
 +
 +    connector.tableOperations().compact(tableName, null, null, true, true);
 +
 +    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
 +
 +    for (Entry<Key,Value> entry : metaScanner) {
 +      String cq = entry.getKey().getColumnQualifier().toString();
 +      Path path = new Path(cq);
 +      Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
 +    }
 +
 +  }
 +
 +  @Test
 +  public void testAddVolumes() throws Exception {
 +
 +    String[] tableNames = getUniqueNames(2);
 +
 +    // grab this before shutting down cluster
 +    String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +    cluster.stop();
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    File v3f = new File(volDirBase, "v3");
 +    v3f.mkdir();
 +    Path v3 = new Path("file://" + v3f.getAbsolutePath());
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v1.toString() + "," + v2.toString() + "," + v3.toString());
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // initialize volume
 +    Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
 +
 +    // check that all volumes are initialized
 +    for (Path volumePath : Arrays.asList(v1, v2, v3)) {
 +      FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
 +      Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
 +      FileStatus[] iids = fs.listStatus(vp);
 +      Assert.assertEquals(1, iids.length);
 +      Assert.assertEquals(uuid, iids[0].getPath().getName());
 +    }
 +
 +    // start cluster and verify that new volume is used
 +    cluster.start();
 +
 +    verifyVolumesUsed(tableNames[1], false, v1, v2, v3);
 +  }
 +
 +  @Test
 +  public void testNonConfiguredVolumes() throws Exception {
 +
 +    String[] tableNames = getUniqueNames(2);
 +
 +    // grab this before shutting down cluster
 +    String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +    cluster.stop();
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    File v3f = new File(volDirBase, "v3");
 +    v3f.mkdir();
 +    Path v3 = new Path("file://" + v3f.getAbsolutePath());
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // initialize volume
 +    Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
 +
 +    // check that all volumes are initialized
 +    for (Path volumePath : Arrays.asList(v1, v2, v3)) {
 +      FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
 +      Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
 +      FileStatus[] iids = fs.listStatus(vp);
 +      Assert.assertEquals(1, iids.length);
 +      Assert.assertEquals(uuid, iids[0].getPath().getName());
 +    }
 +
 +    // start cluster and verify that new volume is used
 +    cluster.start();
 +
 +    // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
 +    List<String> expected = new ArrayList<String>();
 +    for (int i = 0; i < 100; i++) {
 +      String row = String.format("%06d", i * 100 + 3);
 +      expected.add(row + ":cf1:cq1:1");
 +    }
 +
 +    verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
 +
 +    // v1 should not have any data for tableNames[1]
 +    verifyVolumesUsed(tableNames[1], false, v2, v3);
 +  }
 +
 +  private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
 +      MutationsRejectedException {
 +    TreeSet<Text> splits = new TreeSet<Text>();
 +    for (int i = 1; i < 100; i++) {
 +      splits.add(new Text(String.format("%06d", i * 100)));
 +    }
 +
 +    conn.tableOperations().create(tableName);
 +    conn.tableOperations().addSplits(tableName, splits);
 +
 +    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +    for (int i = 0; i < 100; i++) {
 +      String row = String.format("%06d", i * 100 + 3);
 +      Mutation m = new Mutation(row);
 +      m.put("cf1", "cq1", "1");
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +  }
 +
 +  private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
 +      TableExistsException, TableNotFoundException, MutationsRejectedException {
 +
 +    Connector conn = getConnector();
 +
 +    List<String> expected = new ArrayList<String>();
 +    for (int i = 0; i < 100; i++) {
 +      String row = String.format("%06d", i * 100 + 3);
 +      expected.add(row + ":cf1:cq1:1");
 +    }
 +
 +    if (!conn.tableOperations().exists(tableName)) {
 +      Assert.assertFalse(shouldExist);
 +
 +      writeData(tableName, conn);
 +
 +      verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
 +
 +      conn.tableOperations().flush(tableName, null, null, true);
 +    }
 +
 +    verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
 +
 +    String tableId = conn.tableOperations().tableIdMap().get(tableName);
 +    Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
 +    metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
 +    metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +
 +    int counts[] = new int[paths.length];
 +
 +    outer: for (Entry<Key,Value> entry : metaScanner) {
 +      String cf = entry.getKey().getColumnFamily().toString();
 +      String cq = entry.getKey().getColumnQualifier().toString();
 +
 +      String path;
 +      if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
 +        path = cq;
 +      else
 +        path = entry.getValue().toString();
 +
 +      for (int i = 0; i < paths.length; i++) {
 +        if (path.startsWith(paths[i].toString())) {
 +          counts[i]++;
 +          continue outer;
 +        }
 +      }
 +
 +      Assert.fail("Unexpected volume " + path);
 +    }
 +
 +    // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
 +    // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
 +
 +    int sum = 0;
 +    for (int count : counts) {
 +      Assert.assertTrue(count > 0);
 +      sum += count;
 +    }
 +
 +    Assert.assertEquals(200, sum);
 +  }
 +
 +  @Test
 +  public void testRemoveVolumes() throws Exception {
 +    String[] tableNames = getUniqueNames(2);
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +    cluster.stop();
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString());
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // start cluster and verify that volume was decommisioned
 +    cluster.start();
 +
 +    Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
 +    conn.tableOperations().compact(tableNames[0], null, null, true, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v2);
 +
 +    // check that root tablet is not on volume 1
 +    String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
 +    ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
-     String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++    String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
 +    Assert.assertTrue(rootTabletDir.startsWith(v2.toString()));
 +    zookeeper.close();
 +
 +    conn.tableOperations().clone(tableNames[0], tableNames[1], true, new HashMap<String,String>(), new HashSet<String>());
 +
 +    conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
 +    conn.tableOperations().flush(RootTable.NAME, null, null, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v2);
 +    verifyVolumesUsed(tableNames[1], true, v2);
 +
 +  }
 +
 +  private void testReplaceVolume(boolean cleanShutdown) throws Exception {
 +    String[] tableNames = getUniqueNames(3);
 +
 +    verifyVolumesUsed(tableNames[0], false, v1, v2);
 +
 +    // write to 2nd table, but do not flush data to disk before shutdown
 +    writeData(tableNames[1], cluster.getConnector("root", ROOT_PASSWORD));
 +
 +    if (cleanShutdown)
 +      Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 +
 +    cluster.stop();
 +
 +    File v1f = new File(v1.toUri());
 +    File v8f = new File(new File(v1.getParent().toUri()), "v8");
 +    Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
 +    Path v8 = new Path(v8f.toURI());
 +
 +    File v2f = new File(v2.toUri());
 +    File v9f = new File(new File(v2.getParent().toUri()), "v9");
 +    Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
 +    Path v9 = new Path(v9f.toURI());
 +
 +    Configuration conf = new Configuration(false);
 +    conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
 +
 +    conf.set(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
 +    conf.set(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(), v1 + " " + v8 + "," + v2 + " " + v9);
 +    BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
 +    conf.writeXml(fos);
 +    fos.close();
 +
 +    // start cluster and verify that volumes were replaced
 +    cluster.start();
 +
 +    verifyVolumesUsed(tableNames[0], true, v8, v9);
 +    verifyVolumesUsed(tableNames[1], true, v8, v9);
 +
 +    // verify writes to new dir
 +    getConnector().tableOperations().compact(tableNames[0], null, null, true, true);
 +    getConnector().tableOperations().compact(tableNames[1], null, null, true, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v8, v9);
 +    verifyVolumesUsed(tableNames[1], true, v8, v9);
 +
 +    // check that root tablet is not on volume 1 or 2
 +    String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
 +    ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
-     String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++    String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
 +    Assert.assertTrue(rootTabletDir.startsWith(v8.toString()) || rootTabletDir.startsWith(v9.toString()));
 +    zookeeper.close();
 +
 +    getConnector().tableOperations().clone(tableNames[1], tableNames[2], true, new HashMap<String,String>(), new HashSet<String>());
 +
 +    getConnector().tableOperations().flush(MetadataTable.NAME, null, null, true);
 +    getConnector().tableOperations().flush(RootTable.NAME, null, null, true);
 +
 +    verifyVolumesUsed(tableNames[0], true, v8, v9);
 +    verifyVolumesUsed(tableNames[1], true, v8, v9);
 +    verifyVolumesUsed(tableNames[2], true, v8, v9);
 +  }
 +
 +  @Test
 +  public void testCleanReplaceVolumes() throws Exception {
 +    testReplaceVolume(true);
 +  }
 +
 +  @Test
 +  public void testDirtyReplaceVolumes() throws Exception {
 +    testReplaceVolume(false);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
index f657034,0000000..05de342
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
@@@ -1,140 -1,0 +1,141 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collection;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class AddSplitIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void addSplitTest() throws Exception {
 +
 +    String tableName = getUniqueNames(1)[0];
 +    Connector c = getConnector();
 +    c.tableOperations().create(tableName);
 +
 +    insertData(tableName, 1l);
 +
 +    TreeSet<Text> splits = new TreeSet<Text>();
 +    splits.add(new Text(String.format("%09d", 333)));
 +    splits.add(new Text(String.format("%09d", 666)));
 +
 +    c.tableOperations().addSplits(tableName, splits);
 +
 +    UtilWaitThread.sleep(100);
 +
 +    Collection<Text> actualSplits = c.tableOperations().listSplits(tableName);
 +
 +    if (!splits.equals(new TreeSet<Text>(actualSplits))) {
 +      throw new Exception(splits + " != " + actualSplits);
 +    }
 +
 +    verifyData(tableName, 1l);
 +    insertData(tableName, 2l);
 +
 +    // did not clear splits on purpose, it should ignore existing split points
 +    // and still create the three additional split points
 +
 +    splits.add(new Text(String.format("%09d", 200)));
 +    splits.add(new Text(String.format("%09d", 500)));
 +    splits.add(new Text(String.format("%09d", 800)));
 +
 +    c.tableOperations().addSplits(tableName, splits);
 +
 +    UtilWaitThread.sleep(100);
 +
 +    actualSplits = c.tableOperations().listSplits(tableName);
 +
 +    if (!splits.equals(new TreeSet<Text>(actualSplits))) {
 +      throw new Exception(splits + " != " + actualSplits);
 +    }
 +
 +    verifyData(tableName, 2l);
 +  }
 +
 +  private void verifyData(String tableName, long ts) throws Exception {
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +    for (int i = 0; i < 10000; i++) {
 +      if (!iter.hasNext()) {
 +        throw new Exception("row " + i + " not found");
 +      }
 +
 +      Entry<Key,Value> entry = iter.next();
 +
 +      String row = String.format("%09d", i);
 +
 +      if (!entry.getKey().getRow().equals(new Text(row))) {
 +        throw new Exception("unexpected row " + entry.getKey() + " " + i);
 +      }
 +
 +      if (entry.getKey().getTimestamp() != ts) {
 +        throw new Exception("unexpected ts " + entry.getKey() + " " + ts);
 +      }
 +
 +      if (Integer.parseInt(entry.getValue().toString()) != i) {
 +        throw new Exception("unexpected value " + entry + " " + i);
 +      }
 +    }
 +
 +    if (iter.hasNext()) {
 +      throw new Exception("found more than expected " + iter.next());
 +    }
 +
 +  }
 +
 +  private void insertData(String tableName, long ts) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, null);
 +
 +    for (int i = 0; i < 10000; i++) {
 +      String row = String.format("%09d", i);
 +
 +      Mutation m = new Mutation(new Text(row));
-       m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
index 0139a10,0000000..9c4492e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
@@@ -1,110 -1,0 +1,110 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertEquals;
 +
 +import java.util.EnumSet;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BadIteratorMincIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    Connector c = getConnector();
 +
 +    String tableName = getUniqueNames(1)[0];
 +    c.tableOperations().create(tableName);
 +    IteratorSetting is = new IteratorSetting(30, BadIterator.class);
 +    c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
 +    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    Mutation m = new Mutation(new Text("r1"));
-     m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++    m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
 +
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    c.tableOperations().flush(tableName, null, null, false);
 +    UtilWaitThread.sleep(1000);
 +
 +    // minc should fail, so there should be no files
 +    FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 0, 0);
 +
 +    // try to scan table
 +    Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
 +    int count = FunctionalTestUtils.count(scanner);
 +    assertEquals("Did not see expected # entries " + count, 1, count);
 +
 +    // remove the bad iterator
 +    c.tableOperations().removeIterator(tableName, BadIterator.class.getSimpleName(), EnumSet.of(IteratorScope.minc));
 +
 +    UtilWaitThread.sleep(5000);
 +
 +    // minc should complete
 +    FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 1, 1);
 +
 +    count = 0;
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +
 +    if (count != 1)
 +      throw new Exception("Did not see expected # entries " + count);
 +
 +    // now try putting bad iterator back and deleting the table
 +    c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
 +    bw = c.createBatchWriter(tableName, new BatchWriterConfig());
 +    m = new Mutation(new Text("r2"));
-     m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++    m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +    bw.close();
 +
 +    // make sure property is given time to propagate
 +    UtilWaitThread.sleep(500);
 +
 +    c.tableOperations().flush(tableName, null, null, false);
 +
 +    // make sure the flush has time to start
 +    UtilWaitThread.sleep(1000);
 +
 +    // this should not hang
 +    c.tableOperations().delete(tableName);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index c67407d,0000000..688a326
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@@ -1,125 -1,0 +1,126 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BatchScanSplitIT extends ConfigurableMacIT {
 +
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "0"));
 +  }
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void test() throws Exception {
 +    Connector c = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    c.tableOperations().create(tableName);
 +
 +    int numRows = 1 << 18;
 +
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    for (int i = 0; i < numRows; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%09x", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +
 +    getConnector().tableOperations().flush(tableName, null, null, true);
 +
 +    getConnector().tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "4K");
 +
 +    Collection<Text> splits = getConnector().tableOperations().listSplits(tableName);
 +    while (splits.size() < 2) {
 +      UtilWaitThread.sleep(1);
 +      splits = getConnector().tableOperations().listSplits(tableName);
 +    }
 +
 +    System.out.println("splits : " + splits);
 +
 +    Random random = new Random(19011230);
 +    HashMap<Text,Value> expected = new HashMap<Text,Value>();
 +    ArrayList<Range> ranges = new ArrayList<Range>();
 +    for (int i = 0; i < 100; i++) {
 +      int r = random.nextInt(numRows);
 +      Text row = new Text(String.format("%09x", r));
-       expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(Constants.UTF8)));
++      expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(UTF_8)));
 +      ranges.add(new Range(row));
 +    }
 +
 +    // logger.setLevel(Level.TRACE);
 +
 +    HashMap<Text,Value> found = new HashMap<Text,Value>();
 +
 +    for (int i = 0; i < 20; i++) {
 +      BatchScanner bs = getConnector().createBatchScanner(tableName, Authorizations.EMPTY, 4);
 +
 +      found.clear();
 +
 +      long t1 = System.currentTimeMillis();
 +
 +      bs.setRanges(ranges);
 +
 +      for (Entry<Key,Value> entry : bs) {
 +        found.put(entry.getKey().getRow(), entry.getValue());
 +      }
 +      bs.close();
 +
 +      long t2 = System.currentTimeMillis();
 +
 +      log.info(String.format("rate : %06.2f%n", ranges.size() / ((t2 - t1) / 1000.0)));
 +
 +      if (!found.equals(expected))
 +        throw new Exception("Found and expected differ " + found + " " + expected);
 +    }
 +
 +    splits = getConnector().tableOperations().listSplits(tableName);
 +    log.info("splits : " + splits);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
index e2dd903,0000000..465936e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
@@@ -1,181 -1,0 +1,182 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.Random;
 +import java.util.concurrent.TimeUnit;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BatchWriterFlushIT extends SimpleMacIT {
 +
 +  private static final int NUM_TO_FLUSH = 100000;
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 90;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    String[] tableNames = getUniqueNames(2);
 +    String bwft = tableNames[0];
 +    c.tableOperations().create(bwft);
 +    String bwlt = tableNames[1];
 +    c.tableOperations().create(bwlt);
 +    runFlushTest(bwft);
 +    runLatencyTest(bwlt);
 +
 +  }
 +
 +  private void runLatencyTest(String tableName) throws Exception {
 +    // should automatically flush after 2 seconds
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(1000, TimeUnit.MILLISECONDS));
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +
 +    Mutation m = new Mutation(new Text(String.format("r_%10d", 1)));
-     m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(Constants.UTF8)));
++    m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +
 +    UtilWaitThread.sleep(500);
 +
 +    int count = 0;
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +
 +    if (count != 0) {
 +      throw new Exception("Flushed too soon");
 +    }
 +
 +    UtilWaitThread.sleep(1500);
 +
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +
 +    if (count != 1) {
 +      throw new Exception("Did not flush");
 +    }
 +
 +    bw.close();
 +  }
 +
 +  private void runFlushTest(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException,
 +      Exception {
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +    Random r = new Random();
 +
 +    for (int i = 0; i < 4; i++) {
 +      for (int j = 0; j < NUM_TO_FLUSH; j++) {
 +        int row = i * NUM_TO_FLUSH + j;
 +
 +        Mutation m = new Mutation(new Text(String.format("r_%10d", row)));
 +        m.put(new Text("cf"), new Text("cq"), new Value(("" + row).getBytes()));
 +        bw.addMutation(m);
 +      }
 +
 +      bw.flush();
 +
 +      // do a few random lookups into the data just flushed
 +
 +      for (int k = 0; k < 10; k++) {
 +        int rowToLookup = r.nextInt(NUM_TO_FLUSH) + i * NUM_TO_FLUSH;
 +
 +        scanner.setRange(new Range(new Text(String.format("r_%10d", rowToLookup))));
 +
 +        Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +        if (!iter.hasNext())
 +          throw new Exception(" row " + rowToLookup + " not found after flush");
 +
 +        Entry<Key,Value> entry = iter.next();
 +
 +        if (iter.hasNext())
 +          throw new Exception("Scanner returned too much");
 +
 +        verifyEntry(rowToLookup, entry);
 +      }
 +
 +      // scan all data just flushed
 +      scanner.setRange(new Range(new Text(String.format("r_%10d", i * NUM_TO_FLUSH)), true, new Text(String.format("r_%10d", (i + 1) * NUM_TO_FLUSH)), false));
 +      Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +      for (int j = 0; j < NUM_TO_FLUSH; j++) {
 +        int row = i * NUM_TO_FLUSH + j;
 +
 +        if (!iter.hasNext())
 +          throw new Exception("Scan stopped permaturely at " + row);
 +
 +        Entry<Key,Value> entry = iter.next();
 +
 +        verifyEntry(row, entry);
 +      }
 +
 +      if (iter.hasNext())
 +        throw new Exception("Scanner returned too much");
 +
 +    }
 +
 +    bw.close();
 +
 +    // test adding a mutation to a closed batch writer
 +    boolean caught = false;
 +    try {
 +      bw.addMutation(new Mutation(new Text("foobar")));
 +    } catch (IllegalStateException ise) {
 +      caught = true;
 +    }
 +
 +    if (!caught) {
 +      throw new Exception("Adding to closed batch writer did not fail");
 +    }
 +  }
 +
 +  private void verifyEntry(int row, Entry<Key,Value> entry) throws Exception {
 +    if (!entry.getKey().getRow().toString().equals(String.format("r_%10d", row))) {
 +      throw new Exception("Unexpected key returned, expected " + row + " got " + entry.getKey());
 +    }
 +
 +    if (!entry.getValue().toString().equals("" + row)) {
 +      throw new Exception("Unexpected value, expected " + row + " got " + entry.getValue());
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 7974c6c,0000000..ffe55bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@@ -1,120 -1,0 +1,121 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVWriter;
 +import org.apache.accumulo.core.file.rfile.RFile;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.trace.TraceFileSystem;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class BulkFileIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  @Test
 +  public void testBulkFile() throws Exception {
 +    Connector c = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    c.tableOperations().create(tableName);
 +    SortedSet<Text> splits = new TreeSet<Text>();
 +    for (String split : "0333 0666 0999 1333 1666".split(" "))
 +      splits.add(new Text(split));
 +    c.tableOperations().addSplits(tableName, splits);
 +    Configuration conf = new Configuration();
 +    AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
 +    FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
 +
 +    String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
 +
 +    fs.delete(new Path(dir), true);
 +
 +    FileSKVWriter writer1 = FileOperations.getInstance().openWriter(dir + "/f1." + RFile.EXTENSION, fs, conf, aconf);
 +    writer1.startDefaultLocalityGroup();
 +    writeData(writer1, 0, 333);
 +    writer1.close();
 +
 +    FileSKVWriter writer2 = FileOperations.getInstance().openWriter(dir + "/f2." + RFile.EXTENSION, fs, conf, aconf);
 +    writer2.startDefaultLocalityGroup();
 +    writeData(writer2, 334, 999);
 +    writer2.close();
 +
 +    FileSKVWriter writer3 = FileOperations.getInstance().openWriter(dir + "/f3." + RFile.EXTENSION, fs, conf, aconf);
 +    writer3.startDefaultLocalityGroup();
 +    writeData(writer3, 1000, 1999);
 +    writer3.close();
 +
 +    FunctionalTestUtils.bulkImport(c, fs, tableName, dir);
 +
 +    FunctionalTestUtils.checkRFiles(c, tableName, 6, 6, 1, 1);
 +
 +    verifyData(tableName, 0, 1999);
 +
 +  }
 +
 +  private void verifyData(String table, int s, int e) throws Exception {
 +    Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +    for (int i = s; i <= e; i++) {
 +      if (!iter.hasNext())
 +        throw new Exception("row " + i + " not found");
 +
 +      Entry<Key,Value> entry = iter.next();
 +
 +      String row = String.format("%04d", i);
 +
 +      if (!entry.getKey().getRow().equals(new Text(row)))
 +        throw new Exception("unexpected row " + entry.getKey() + " " + i);
 +
 +      if (Integer.parseInt(entry.getValue().toString()) != i)
 +        throw new Exception("unexpected value " + entry + " " + i);
 +    }
 +
 +    if (iter.hasNext())
 +      throw new Exception("found more than expected " + iter.next());
 +  }
 +
 +  private void writeData(FileSKVWriter w, int s, int e) throws Exception {
 +    for (int i = s; i <= e; i++) {
-       w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(UTF_8)));
 +    }
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
index 785a997,0000000..92bd714
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
@@@ -1,147 -1,0 +1,148 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.EnumSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableExistsException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ConcurrencyIT extends ConfigurableMacIT {
 +  
 +  static class ScanTask extends Thread {
 +    
 +    int count = 0;
 +    Scanner scanner;
 +    
 +    ScanTask(Connector conn, long time) throws Exception {
 +      scanner = conn.createScanner("cct", Authorizations.EMPTY);
 +      IteratorSetting slow = new IteratorSetting(30, "slow", SlowIterator.class);
 +      SlowIterator.setSleepTime(slow, time);
 +      scanner.addScanIterator(slow);
 +    }
 +    
 +    @Override
 +    public void run() {
 +      count = FunctionalTestUtils.count(scanner);
 +    }
 +    
 +  }
 +  
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1"));
 +  }
 +  
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 2 * 60;
 +  }
 +
 +  /*
 +   * Below is a diagram of the operations in this test over time.
 +   * 
 +   * Scan 0 |------------------------------| Scan 1 |----------| Minc 1 |-----| Scan 2 |----------| Scan 3 |---------------| Minc 2 |-----| Majc 1 |-----|
 +   */
 +  
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    runTest(c);
 +  }
 +
 +  static void runTest(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
 +      MutationsRejectedException, Exception, InterruptedException {
 +    c.tableOperations().create("cct");
 +    IteratorSetting is = new IteratorSetting(10, SlowIterator.class);
 +    SlowIterator.setSleepTime(is, 50);
 +    c.tableOperations().attachIterator("cct", is, EnumSet.of(IteratorScope.minc, IteratorScope.majc));
 +    c.tableOperations().setProperty("cct", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
 +    
 +    BatchWriter bw = c.createBatchWriter("cct", new BatchWriterConfig());
 +    for (int i = 0; i < 50; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%06d", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +    bw.flush();
 +    
 +    ScanTask st0 = new ScanTask(c, 300);
 +    st0.start();
 +    
 +    ScanTask st1 = new ScanTask(c, 100);
 +    st1.start();
 +    
 +    UtilWaitThread.sleep(50);
 +    c.tableOperations().flush("cct", null, null, true);
 +    
 +    for (int i = 0; i < 50; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%06d", i)));
-       m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++      m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
 +      bw.addMutation(m);
 +    }
 +    
 +    bw.flush();
 +    
 +    ScanTask st2 = new ScanTask(c, 100);
 +    st2.start();
 +    
 +    st1.join();
 +    st2.join();
 +    if (st1.count != 50)
 +      throw new Exception("Thread 1 did not see 50, saw " + st1.count);
 +    
 +    if (st2.count != 50)
 +      throw new Exception("Thread 2 did not see 50, saw " + st2.count);
 +    
 +    ScanTask st3 = new ScanTask(c, 150);
 +    st3.start();
 +    
 +    UtilWaitThread.sleep(50);
 +    c.tableOperations().flush("cct", null, null, false);
 +    
 +    st3.join();
 +    if (st3.count != 50)
 +      throw new Exception("Thread 3 did not see 50, saw " + st3.count);
 +    
 +    st0.join();
 +    if (st0.count != 50)
 +      throw new Exception("Thread 0 did not see 50, saw " + st0.count);
 +    
 +    bw.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
index d5e9188,0000000..7e5944e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@@ -1,317 -1,0 +1,318 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.HashMap;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.ConstraintViolationSummary;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint;
 +import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class ConstraintIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 30;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    String[] tableNames = getUniqueNames(3);
 +    Connector c = getConnector();
 +    for (String table : tableNames) {
 +      c.tableOperations().create(table);
 +      c.tableOperations().addConstraint(table, NumericValueConstraint.class.getName());
 +      c.tableOperations().addConstraint(table, AlphaNumKeyConstraint.class.getName());
 +    }
 +
 +    // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
 +    // logger.setLevel(Level.TRACE);
 +
 +    test1(tableNames[0]);
 +
 +    // logger.setLevel(Level.TRACE);
 +
 +    test2(tableNames[1], false);
 +    test2(tableNames[2], true);
 +  }
 +
 +  private void test1(String tableName) throws Exception {
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    Mutation mut1 = new Mutation(new Text("r1"));
-     mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(Constants.UTF8)));
++    mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(UTF_8)));
 +
 +    bw.addMutation(mut1);
 +
 +    // should not throw any exceptions
 +    bw.close();
 +
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    // create a mutation with a non numeric value
 +    Mutation mut2 = new Mutation(new Text("r1"));
-     mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(Constants.UTF8)));
++    mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(UTF_8)));
 +
 +    bw.addMutation(mut2);
 +
 +    boolean sawMRE = false;
 +
 +    try {
 +      bw.close();
 +      // should not get here
 +      throw new Exception("Test failed, constraint did not catch bad mutation");
 +    } catch (MutationsRejectedException mre) {
 +      sawMRE = true;
 +
 +      // verify constraint violation summary
 +      List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
 +
 +      if (cvsl.size() != 1) {
 +        throw new Exception("Unexpected constraints");
 +      }
 +
 +      for (ConstraintViolationSummary cvs : cvsl) {
 +        if (!cvs.constrainClass.equals(NumericValueConstraint.class.getName())) {
 +          throw new Exception("Unexpected constraint class " + cvs.constrainClass);
 +        }
 +
 +        if (cvs.numberOfViolatingMutations != 1) {
 +          throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
 +        }
 +      }
 +    }
 +
 +    if (!sawMRE) {
 +      throw new Exception("Did not see MutationsRejectedException");
 +    }
 +
 +    // verify mutation did not go through
 +    Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new Range(new Text("r1")));
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +    Entry<Key,Value> entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    // remove the numeric value constraint
 +    getConnector().tableOperations().removeConstraint(tableName, 2);
 +    UtilWaitThread.sleep(1000);
 +
 +    // now should be able to add a non numeric value
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +    bw.addMutation(mut2);
 +    bw.close();
 +
 +    // verify mutation went through
 +    iter = scanner.iterator();
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    // add a constraint that references a non-existant class
 +    getConnector().tableOperations().setProperty(tableName, Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistantClass");
 +    UtilWaitThread.sleep(1000);
 +
 +    // add a mutation
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    Mutation mut3 = new Mutation(new Text("r1"));
-     mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++    mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
 +
 +    bw.addMutation(mut3);
 +
 +    sawMRE = false;
 +
 +    try {
 +      bw.close();
 +      // should not get here
 +      throw new Exception("Test failed, mutation went through when table had bad constraints");
 +    } catch (MutationsRejectedException mre) {
 +      sawMRE = true;
 +    }
 +
 +    if (!sawMRE) {
 +      throw new Exception("Did not see MutationsRejectedException");
 +    }
 +
 +    // verify the mutation did not go through
 +    iter = scanner.iterator();
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    // remove the bad constraint
 +    getConnector().tableOperations().removeConstraint(tableName, 1);
 +    UtilWaitThread.sleep(1000);
 +
 +    // try the mutation again
 +    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +    bw.addMutation(mut3);
 +    bw.close();
 +
 +    // verify it went through
 +    iter = scanner.iterator();
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +  }
 +
 +  private Mutation newMut(String row, String cf, String cq, String val) {
 +    Mutation mut1 = new Mutation(new Text(row));
-     mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(Constants.UTF8)));
++    mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(UTF_8)));
 +    return mut1;
 +  }
 +
 +  private void test2(String table, boolean doFlush) throws Exception {
 +    // test sending multiple mutations with multiple constrain violations... all of the non violating mutations
 +    // should go through
 +    int numericErrors = 2;
 +
 +    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
 +    bw.addMutation(newMut("r1", "cf1", "cq1", "123"));
 +    bw.addMutation(newMut("r1", "cf1", "cq2", "I'm a bad value"));
 +    if (doFlush) {
 +      try {
 +        bw.flush();
 +        throw new Exception("Didn't find a bad mutation");
 +      } catch (MutationsRejectedException mre) {
 +        // ignored
 +        try {
 +          bw.close();
 +        } catch (MutationsRejectedException ex) {
 +          // ignored
 +        }
 +        bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
 +        numericErrors = 1;
 +      }
 +    }
 +    bw.addMutation(newMut("r1", "cf1", "cq3", "I'm a naughty value"));
 +    bw.addMutation(newMut("@bad row@", "cf1", "cq2", "456"));
 +    bw.addMutation(newMut("r1", "cf1", "cq4", "789"));
 +
 +    boolean sawMRE = false;
 +
 +    try {
 +      bw.close();
 +      // should not get here
 +      throw new Exception("Test failed, constraint did not catch bad mutation");
 +    } catch (MutationsRejectedException mre) {
 +      System.out.println(mre);
 +
 +      sawMRE = true;
 +
 +      // verify constraint violation summary
 +      List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
 +
 +      if (cvsl.size() != 2) {
 +        throw new Exception("Unexpected constraints");
 +      }
 +
 +      HashMap<String,Integer> expected = new HashMap<String,Integer>();
 +
 +      expected.put("org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", numericErrors);
 +      expected.put("org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint", 1);
 +
 +      for (ConstraintViolationSummary cvs : cvsl) {
 +        if (expected.get(cvs.constrainClass) != cvs.numberOfViolatingMutations) {
 +          throw new Exception("Unexpected " + cvs.constrainClass + " " + cvs.numberOfViolatingMutations);
 +        }
 +      }
 +    }
 +
 +    if (!sawMRE) {
 +      throw new Exception("Did not see MutationsRejectedException");
 +    }
 +
 +    Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
 +
 +    Iterator<Entry<Key,Value>> iter = scanner.iterator();
 +
 +    Entry<Key,Value> entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    entry = iter.next();
 +
 +    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
-         || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(Constants.UTF8)))) {
++        || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(UTF_8)))) {
 +      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +    if (iter.hasNext()) {
 +      entry = iter.next();
 +      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
 +    }
 +
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
index b9b54c7,0000000..5b5249b
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
@@@ -1,128 -1,0 +1,129 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.ArrayList;
 +import java.util.Iterator;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Assert;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +
 +import com.google.common.collect.Iterators;
 +
 +public class CreateAndUseIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 4 * 60;
 +  }
 +
 +  private static SortedSet<Text> splits;
 +
 +  @BeforeClass
 +  public static void createData() throws Exception {
 +    splits = new TreeSet<Text>();
 +
 +    for (int i = 1; i < 256; i++) {
 +      splits.add(new Text(String.format("%08x", i << 8)));
 +    }
 +  }
 +
 +  @Test
 +  public void verifyDataIsPresent() throws Exception {
 +    Text cf = new Text("cf1");
 +    Text cq = new Text("cq1");
 +
 +    String tableName = getUniqueNames(1)[0];
 +    getConnector().tableOperations().create(tableName);
 +    getConnector().tableOperations().addSplits(tableName, splits);
 +    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    for (int i = 1; i < 257; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%08x", (i << 8) - 16)));
-       m.put(cf, cq, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++      m.put(cf, cq, new Value(Integer.toString(i).getBytes(UTF_8)));
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +    Scanner scanner1 = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +
 +    int ei = 1;
 +
 +    for (Entry<Key,Value> entry : scanner1) {
 +      Assert.assertEquals(String.format("%08x", (ei << 8) - 16), entry.getKey().getRow().toString());
 +      Assert.assertEquals(Integer.toString(ei), entry.getValue().toString());
 +
 +      ei++;
 +    }
 +
 +    Assert.assertEquals("Did not see expected number of rows", 257, ei);
 +  }
 +
 +  @Test
 +  public void createTableAndScan() throws Exception {
 +    String table2 = getUniqueNames(1)[0];
 +    getConnector().tableOperations().create(table2);
 +    getConnector().tableOperations().addSplits(table2, splits);
 +    Scanner scanner2 = getConnector().createScanner(table2, Authorizations.EMPTY);
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner2) {
 +      if (entry != null)
 +        count++;
 +    }
 +
 +    if (count != 0) {
 +      throw new Exception("Did not see expected number of entries, count = " + count);
 +    }
 +  }
 +
 +  @Test
 +  public void createTableAndBatchScan() throws Exception {
 +    ArrayList<Range> ranges = new ArrayList<Range>();
 +    for (int i = 1; i < 257; i++) {
 +      ranges.add(new Range(new Text(String.format("%08x", (i << 8) - 16))));
 +    }
 +
 +    String table3 = getUniqueNames(1)[0];
 +    getConnector().tableOperations().create(table3);
 +    getConnector().tableOperations().addSplits(table3, splits);
 +    BatchScanner bs = getConnector().createBatchScanner(table3, Authorizations.EMPTY, 3);
 +    bs.setRanges(ranges);
 +    Iterator<Entry<Key,Value>> iter = bs.iterator();
 +    int count = Iterators.size(iter);
 +    bs.close();
 +
 +    Assert.assertEquals("Did not expect to find any entries", 0, count);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
index 83b5b24,0000000..0578ef4
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
@@@ -1,94 -1,0 +1,94 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.*;
 +
 +import java.util.Collections;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class DeleteEverythingIT extends ConfigurableMacIT {
 +  
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1s"));
 +  }
 +  
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    Connector c = getConnector();
 +    c.tableOperations().create("de");
 +    BatchWriter bw = getConnector().createBatchWriter("de", new BatchWriterConfig());
 +    Mutation m = new Mutation(new Text("foo"));
-     m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(Constants.UTF8)));
++    m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(UTF_8)));
 +    bw.addMutation(m);
 +    bw.flush();
 +    
 +    getConnector().tableOperations().flush("de", null, null, true);
 +    
 +    FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 1, 1);
 +    
 +    m = new Mutation(new Text("foo"));
 +    m.putDelete(new Text("bar"), new Text("1910"));
 +    bw.addMutation(m);
 +    bw.flush();
 +    
 +    Scanner scanner = getConnector().createScanner("de", Authorizations.EMPTY);
 +    scanner.setRange(new Range());
 +    int count = FunctionalTestUtils.count(scanner);
 +    assertEquals("count == " + count, 0, count);
 +    getConnector().tableOperations().flush("de", null, null, true);
 +    
 +    getConnector().tableOperations().setProperty("de", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
 +    UtilWaitThread.sleep(4000);
 +    
 +    FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 0, 0);
 +    
 +    bw.close();
 +    
 +    count = 0;
 +    for (@SuppressWarnings("unused")
 +    Entry<Key,Value> entry : scanner) {
 +      count++;
 +    }
 +    
 +    if (count != 0)
 +      throw new Exception("count == " + count);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
index a8237a6,0000000..d35ba9f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
@@@ -1,143 -1,0 +1,143 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.functional;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.Map.Entry;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.junit.Test;
 +
 +// attempt to reproduce ACCUMULO-315
 +public class DeleteRowsSplitIT extends SimpleMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 4 * 60;
 +  }
 +
 +  private static final Logger log = Logger.getLogger(DeleteRowsSplitIT.class);
 +
 +  private static final String LETTERS = "abcdefghijklmnopqrstuvwxyz";
 +  static final SortedSet<Text> SPLITS = new TreeSet<Text>();
 +  static final List<String> ROWS = new ArrayList<String>();
 +  static {
-     for (byte b : LETTERS.getBytes(Constants.UTF8)) {
++    for (byte b : LETTERS.getBytes(UTF_8)) {
 +      SPLITS.add(new Text(new byte[] {b}));
-       ROWS.add(new String(new byte[] {b}, Constants.UTF8));
++      ROWS.add(new String(new byte[] {b}, UTF_8));
 +    }
 +  }
 +
 +  @Test
 +  public void run() throws Exception {
 +    // Delete ranges of rows, and verify the are removed
 +    // Do this while adding many splits
 +    final String tableName = getUniqueNames(1)[0];
 +
 +    // Eliminate whole tablets
 +    for (int test = 0; test < 10; test++) {
 +      // create a table
 +      log.info("Test " + test);
 +      getConnector().tableOperations().create(tableName);
 +
 +      // put some data in it
 +      fillTable(tableName);
 +
 +      // generate a random delete range
 +      final Text start = new Text();
 +      final Text end = new Text();
 +      generateRandomRange(start, end);
 +
 +      // initiate the delete range
 +      final boolean fail[] = {false};
 +      Thread t = new Thread() {
 +        @Override
 +        public void run() {
 +          try {
 +            // split the table
 +            final SortedSet<Text> afterEnd = SPLITS.tailSet(new Text(end.toString() + "\0"));
 +            getConnector().tableOperations().addSplits(tableName, afterEnd);
 +          } catch (Exception ex) {
 +            log.error(ex, ex);
 +            synchronized (fail) {
 +              fail[0] = true;
 +            }
 +          }
 +        }
 +      };
 +      t.start();
 +
 +      UtilWaitThread.sleep(test * 2);
 +
 +      getConnector().tableOperations().deleteRows(tableName, start, end);
 +
 +      t.join();
 +      synchronized (fail) {
 +        assertTrue(!fail[0]);
 +      }
 +
 +      // scan the table
 +      Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
 +      for (Entry<Key,Value> entry : scanner) {
 +        Text row = entry.getKey().getRow();
 +        assertTrue(row.compareTo(start) <= 0 || row.compareTo(end) > 0);
 +      }
 +
 +      // delete the table
 +      getConnector().tableOperations().delete(tableName);
 +    }
 +  }
 +
 +  private void generateRandomRange(Text start, Text end) {
 +    List<String> bunch = new ArrayList<String>(ROWS);
 +    Collections.shuffle(bunch);
 +    if (bunch.get(0).compareTo((bunch.get(1))) < 0) {
 +      start.set(bunch.get(0));
 +      end.set(bunch.get(1));
 +    } else {
 +      start.set(bunch.get(1));
 +      end.set(bunch.get(0));
 +    }
 +
 +  }
 +
 +  private void fillTable(String table) throws Exception {
 +    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
 +    for (String row : ROWS) {
 +      Mutation m = new Mutation(row);
 +      m.put("cf", "cq", "value");
 +      bw.addMutation(m);
 +    }
 +    bw.close();
 +  }
 +}


[27/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index 0778f5b,0000000..3459069
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@@ -1,3857 -1,0 +1,3859 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.DataInputStream;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.EnumSet;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.PriorityQueue;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicLong;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.concurrent.locks.ReentrantLock;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.ConfigurationCopy;
 +import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Violations;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Column;
 +import org.apache.accumulo.core.data.ColumnUpdate;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.KeyValue;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.MapFileInfo;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVIterator;
 +import org.apache.accumulo.core.iterators.IterationInterruptedException;
 +import org.apache.accumulo.core.iterators.IteratorEnvironment;
 +import org.apache.accumulo.core.iterators.IteratorUtil;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
 +import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
 +import org.apache.accumulo.core.iterators.system.DeletingIterator;
 +import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
 +import org.apache.accumulo.core.iterators.system.MultiIterator;
 +import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
 +import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSource;
 +import org.apache.accumulo.core.iterators.system.StatsIterator;
 +import org.apache.accumulo.core.iterators.system.VisibilityFilter;
 +import org.apache.accumulo.core.master.thrift.TabletLoadState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.LocalityGroupUtil;
 +import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.tableOps.CompactionIterators;
 +import org.apache.accumulo.server.problems.ProblemReport;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.problems.ProblemType;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.tablets.TabletTime;
 +import org.apache.accumulo.server.tablets.UniqueNameAllocator;
 +import org.apache.accumulo.server.util.FileUtil;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.TabletOperations;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.instrument.Trace;
 +import org.apache.accumulo.tserver.Compactor.CompactionCanceledException;
 +import org.apache.accumulo.tserver.Compactor.CompactionEnv;
 +import org.apache.accumulo.tserver.FileManager.ScanFileManager;
 +import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
 +import org.apache.accumulo.tserver.TabletServer.TservConstraintEnv;
 +import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
 +import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
 +import org.apache.accumulo.tserver.compaction.CompactionPlan;
 +import org.apache.accumulo.tserver.compaction.CompactionStrategy;
 +import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
 +import org.apache.accumulo.tserver.compaction.WriteParameters;
 +import org.apache.accumulo.tserver.constraints.ConstraintChecker;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.MutationReceiver;
 +import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
 +import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
 +import org.apache.commons.codec.DecoderException;
 +import org.apache.commons.codec.binary.Hex;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +import com.google.common.annotations.VisibleForTesting;
 +
 +/*
 + * We need to be able to have the master tell a tabletServer to
 + * close this file, and the tablet server to handle all pending client reads
 + * before closing
 + *
 + */
 +
 +/**
 + *
 + * this class just provides an interface to read from a MapFile mostly takes care of reporting start and end keys
 + *
 + * need this because a single row extent can have multiple columns this manages all the columns (each handled by a store) for a single row-extent
 + *
 + *
 + */
 +
 +public class Tablet {
 +
 +  enum MinorCompactionReason {
 +    USER, SYSTEM, CLOSE, RECOVERY
 +  }
 +
 +  public class CommitSession {
 +
 +    private int seq;
 +    private InMemoryMap memTable;
 +    private int commitsInProgress;
 +    private long maxCommittedTime = Long.MIN_VALUE;
 +
 +    private CommitSession(int seq, InMemoryMap imm) {
 +      this.seq = seq;
 +      this.memTable = imm;
 +      commitsInProgress = 0;
 +    }
 +
 +    public int getWALogSeq() {
 +      return seq;
 +    }
 +
 +    private void decrementCommitsInProgress() {
 +      if (commitsInProgress < 1)
 +        throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
 +
 +      commitsInProgress--;
 +      if (commitsInProgress == 0)
 +        Tablet.this.notifyAll();
 +    }
 +
 +    private void incrementCommitsInProgress() {
 +      if (commitsInProgress < 0)
 +        throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
 +
 +      commitsInProgress++;
 +    }
 +
 +    private void waitForCommitsToFinish() {
 +      while (commitsInProgress > 0) {
 +        try {
 +          Tablet.this.wait(50);
 +        } catch (InterruptedException e) {
 +          log.warn(e, e);
 +        }
 +      }
 +    }
 +
 +    public void abortCommit(List<Mutation> value) {
 +      Tablet.this.abortCommit(this, value);
 +    }
 +
 +    public void commit(List<Mutation> mutations) {
 +      Tablet.this.commit(this, mutations);
 +    }
 +
 +    public Tablet getTablet() {
 +      return Tablet.this;
 +    }
 +
 +    public boolean beginUpdatingLogsUsed(ArrayList<DfsLogger> copy, boolean mincFinish) {
 +      return Tablet.this.beginUpdatingLogsUsed(memTable, copy, mincFinish);
 +    }
 +
 +    public void finishUpdatingLogsUsed() {
 +      Tablet.this.finishUpdatingLogsUsed();
 +    }
 +
 +    public int getLogId() {
 +      return logId;
 +    }
 +
 +    public KeyExtent getExtent() {
 +      return extent;
 +    }
 +
 +    private void updateMaxCommittedTime(long time) {
 +      maxCommittedTime = Math.max(time, maxCommittedTime);
 +    }
 +
 +    private long getMaxCommittedTime() {
 +      if (maxCommittedTime == Long.MIN_VALUE)
 +        throw new IllegalStateException("Tried to read max committed time when it was never set");
 +      return maxCommittedTime;
 +    }
 +
 +  }
 +
 +  private class TabletMemory {
 +    private InMemoryMap memTable;
 +    private InMemoryMap otherMemTable;
 +    private InMemoryMap deletingMemTable;
 +    private int nextSeq = 1;
 +    private CommitSession commitSession;
 +
 +    TabletMemory() {
 +      try {
 +        memTable = new InMemoryMap(acuTableConf);
 +      } catch (LocalityGroupConfigurationError e) {
 +        throw new RuntimeException(e);
 +      }
 +      commitSession = new CommitSession(nextSeq, memTable);
 +      nextSeq += 2;
 +    }
 +
 +    InMemoryMap getMemTable() {
 +      return memTable;
 +    }
 +
 +    InMemoryMap getMinCMemTable() {
 +      return otherMemTable;
 +    }
 +
 +    CommitSession prepareForMinC() {
 +      if (otherMemTable != null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      if (deletingMemTable != null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      otherMemTable = memTable;
 +      try {
 +        memTable = new InMemoryMap(acuTableConf);
 +      } catch (LocalityGroupConfigurationError e) {
 +        throw new RuntimeException(e);
 +      }
 +
 +      CommitSession oldCommitSession = commitSession;
 +      commitSession = new CommitSession(nextSeq, memTable);
 +      nextSeq += 2;
 +
 +      tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), otherMemTable.estimatedSizeInBytes());
 +
 +      return oldCommitSession;
 +    }
 +
 +    void finishedMinC() {
 +
 +      if (otherMemTable == null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      if (deletingMemTable != null) {
 +        throw new IllegalStateException();
 +      }
 +
 +      deletingMemTable = otherMemTable;
 +
 +      otherMemTable = null;
 +      Tablet.this.notifyAll();
 +    }
 +
 +    void finalizeMinC() {
 +      try {
 +        deletingMemTable.delete(15000);
 +      } finally {
 +        synchronized (Tablet.this) {
 +          if (otherMemTable != null) {
 +            throw new IllegalStateException();
 +          }
 +
 +          if (deletingMemTable == null) {
 +            throw new IllegalStateException();
 +          }
 +
 +          deletingMemTable = null;
 +
 +          tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), 0);
 +        }
 +      }
 +    }
 +
 +    boolean memoryReservedForMinC() {
 +      return otherMemTable != null || deletingMemTable != null;
 +    }
 +
 +    void waitForMinC() {
 +      while (otherMemTable != null || deletingMemTable != null) {
 +        try {
 +          Tablet.this.wait(50);
 +        } catch (InterruptedException e) {
 +          log.warn(e, e);
 +        }
 +      }
 +    }
 +
 +    void mutate(CommitSession cm, List<Mutation> mutations) {
 +      cm.memTable.mutate(mutations);
 +    }
 +
 +    void updateMemoryUsageStats() {
 +      long other = 0;
 +      if (otherMemTable != null)
 +        other = otherMemTable.estimatedSizeInBytes();
 +      else if (deletingMemTable != null)
 +        other = deletingMemTable.estimatedSizeInBytes();
 +
 +      tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), other);
 +    }
 +
 +    List<MemoryIterator> getIterators() {
 +      List<MemoryIterator> toReturn = new ArrayList<MemoryIterator>(2);
 +      toReturn.add(memTable.skvIterator());
 +      if (otherMemTable != null)
 +        toReturn.add(otherMemTable.skvIterator());
 +      return toReturn;
 +    }
 +
 +    void returnIterators(List<MemoryIterator> iters) {
 +      for (MemoryIterator iter : iters) {
 +        iter.close();
 +      }
 +    }
 +
 +    public long getNumEntries() {
 +      if (otherMemTable != null)
 +        return memTable.getNumEntries() + otherMemTable.getNumEntries();
 +      return memTable.getNumEntries();
 +    }
 +
 +    CommitSession getCommitSession() {
 +      return commitSession;
 +    }
 +  }
 +
 +  private TabletMemory tabletMemory;
 +
 +  private final TabletTime tabletTime;
 +  private long persistedTime;
 +  private final Object timeLock = new Object();
 +
 +  private final Path location; // absolute path of this tablets dir
 +  private TServerInstance lastLocation;
 +
 +  private Configuration conf;
 +  private VolumeManager fs;
 +
 +  private TableConfiguration acuTableConf;
 +
 +  private volatile boolean tableDirChecked = false;
 +
 +  private AtomicLong dataSourceDeletions = new AtomicLong(0);
 +  private Set<ScanDataSource> activeScans = new HashSet<ScanDataSource>();
 +
 +  private volatile boolean closing = false;
 +  private boolean closed = false;
 +  private boolean closeComplete = false;
 +
 +  private long lastFlushID = -1;
 +  private long lastCompactID = -1;
 +
 +  private static class CompactionWaitInfo {
 +    long flushID = -1;
 +    long compactionID = -1;
 +  }
 +
 +  // stores info about user initiated major compaction that is waiting on a minor compaction to finish
 +  private CompactionWaitInfo compactionWaitInfo = new CompactionWaitInfo();
 +
 +  private KeyExtent extent;
 +
 +  private TabletResourceManager tabletResources;
 +  final private DatafileManager datafileManager;
 +  private volatile boolean majorCompactionInProgress = false;
 +  private volatile boolean majorCompactionWaitingToStart = false;
 +  private Set<MajorCompactionReason> majorCompactionQueued = Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
 +  private volatile boolean minorCompactionInProgress = false;
 +  private volatile boolean minorCompactionWaitingToStart = false;
 +
 +  private boolean updatingFlushID = false;
 +
 +  private AtomicReference<ConstraintChecker> constraintChecker = new AtomicReference<ConstraintChecker>();
 +
 +  private final String tabletDirectory;
 +
 +  private int writesInProgress = 0;
 +
 +  private static final Logger log = Logger.getLogger(Tablet.class);
 +  public TabletStatsKeeper timer;
 +
 +  private Rate queryRate = new Rate(0.2);
 +  private long queryCount = 0;
 +
 +  private Rate queryByteRate = new Rate(0.2);
 +  private long queryBytes = 0;
 +
 +  private Rate ingestRate = new Rate(0.2);
 +  private long ingestCount = 0;
 +
 +  private Rate ingestByteRate = new Rate(0.2);
 +  private long ingestBytes = 0;
 +
 +  private byte[] defaultSecurityLabel = new byte[0];
 +
 +  private long lastMinorCompactionFinishTime;
 +  private long lastMapFileImportTime;
 +
 +  private volatile long numEntries;
 +  private volatile long numEntriesInMemory;
 +
 +  // a count of the amount of data read by the iterators
 +  private AtomicLong scannedCount = new AtomicLong(0);
 +  private Rate scannedRate = new Rate(0.2);
 +
 +  private ConfigurationObserver configObserver;
 +
 +  private TabletServer tabletServer;
 +
 +  private final int logId;
 +  // ensure we only have one reader/writer of our bulk file notes at at time
 +  public final Object bulkFileImportLock = new Object();
 +
 +  public int getLogId() {
 +    return logId;
 +  }
 +
 +  public static class TabletClosedException extends RuntimeException {
 +    public TabletClosedException(Exception e) {
 +      super(e);
 +    }
 +
 +    public TabletClosedException() {
 +      super();
 +    }
 +
 +    private static final long serialVersionUID = 1L;
 +  }
 +
 +  FileRef getNextMapFilename(String prefix) throws IOException {
 +    String extension = FileOperations.getNewFileExtension(tabletServer.getTableConfiguration(extent));
 +    checkTabletDir();
 +    return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension);
 +  }
 +
 +  private void checkTabletDir() throws IOException {
 +    if (!tableDirChecked) {
 +      checkTabletDir(this.location);
 +      tableDirChecked = true;
 +    }
 +  }
 +
 +  private void checkTabletDir(Path tabletDir) throws IOException {
 +
 +    FileStatus[] files = null;
 +    try {
 +      files = fs.listStatus(tabletDir);
 +    } catch (FileNotFoundException ex) {
 +      // ignored
 +    }
 +
 +    if (files == null) {
 +      if (tabletDir.getName().startsWith(Constants.CLONE_PREFIX))
 +        log.debug("Tablet " + extent + " had no dir, creating " + tabletDir); // its a clone dir...
 +      else
 +        log.warn("Tablet " + extent + " had no dir, creating " + tabletDir);
 +
 +      fs.mkdirs(tabletDir);
 +    }
 +  }
 +
 +  class DatafileManager {
 +    // access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
 +    final private Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
 +
 +    DatafileManager(SortedMap<FileRef,DataFileValue> datafileSizes) {
 +      for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet())
 +        this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
 +    }
 +
 +    FileRef mergingMinorCompactionFile = null;
 +    Set<FileRef> filesToDeleteAfterScan = new HashSet<FileRef>();
 +    Map<Long,Set<FileRef>> scanFileReservations = new HashMap<Long,Set<FileRef>>();
 +    MapCounter<FileRef> fileScanReferenceCounts = new MapCounter<FileRef>();
 +    long nextScanReservationId = 0;
 +    boolean reservationsBlocked = false;
 +
 +    Set<FileRef> majorCompactingFiles = new HashSet<FileRef>();
 +
 +    Pair<Long,Map<FileRef,DataFileValue>> reserveFilesForScan() {
 +      synchronized (Tablet.this) {
 +
 +        while (reservationsBlocked) {
 +          try {
 +            Tablet.this.wait(50);
 +          } catch (InterruptedException e) {
 +            log.warn(e, e);
 +          }
 +        }
 +
 +        Set<FileRef> absFilePaths = new HashSet<FileRef>(datafileSizes.keySet());
 +
 +        long rid = nextScanReservationId++;
 +
 +        scanFileReservations.put(rid, absFilePaths);
 +
 +        Map<FileRef,DataFileValue> ret = new HashMap<FileRef,DataFileValue>();
 +
 +        for (FileRef path : absFilePaths) {
 +          fileScanReferenceCounts.increment(path, 1);
 +          ret.put(path, datafileSizes.get(path));
 +        }
 +
 +        return new Pair<Long,Map<FileRef,DataFileValue>>(rid, ret);
 +      }
 +    }
 +
 +    void returnFilesForScan(Long reservationId) {
 +
 +      final Set<FileRef> filesToDelete = new HashSet<FileRef>();
 +
 +      synchronized (Tablet.this) {
 +        Set<FileRef> absFilePaths = scanFileReservations.remove(reservationId);
 +
 +        if (absFilePaths == null)
 +          throw new IllegalArgumentException("Unknown scan reservation id " + reservationId);
 +
 +        boolean notify = false;
 +        for (FileRef path : absFilePaths) {
 +          long refCount = fileScanReferenceCounts.decrement(path, 1);
 +          if (refCount == 0) {
 +            if (filesToDeleteAfterScan.remove(path))
 +              filesToDelete.add(path);
 +            notify = true;
 +          } else if (refCount < 0)
 +            throw new IllegalStateException("Scan ref count for " + path + " is " + refCount);
 +        }
 +
 +        if (notify)
 +          Tablet.this.notifyAll();
 +      }
 +
 +      if (filesToDelete.size() > 0) {
 +        log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
 +        MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
 +      }
 +    }
 +
 +    private void removeFilesAfterScan(Set<FileRef> scanFiles) {
 +      if (scanFiles.size() == 0)
 +        return;
 +
 +      Set<FileRef> filesToDelete = new HashSet<FileRef>();
 +
 +      synchronized (Tablet.this) {
 +        for (FileRef path : scanFiles) {
 +          if (fileScanReferenceCounts.get(path) == 0)
 +            filesToDelete.add(path);
 +          else
 +            filesToDeleteAfterScan.add(path);
 +        }
 +      }
 +
 +      if (filesToDelete.size() > 0) {
 +        log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
 +        MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
 +      }
 +    }
 +
 +    private TreeSet<FileRef> waitForScansToFinish(Set<FileRef> pathsToWaitFor, boolean blockNewScans, long maxWaitTime) {
 +      long startTime = System.currentTimeMillis();
 +      TreeSet<FileRef> inUse = new TreeSet<FileRef>();
 +
 +      Span waitForScans = Trace.start("waitForScans");
 +      try {
 +        synchronized (Tablet.this) {
 +          if (blockNewScans) {
 +            if (reservationsBlocked)
 +              throw new IllegalStateException();
 +
 +            reservationsBlocked = true;
 +          }
 +
 +          for (FileRef path : pathsToWaitFor) {
 +            while (fileScanReferenceCounts.get(path) > 0 && System.currentTimeMillis() - startTime < maxWaitTime) {
 +              try {
 +                Tablet.this.wait(100);
 +              } catch (InterruptedException e) {
 +                log.warn(e, e);
 +              }
 +            }
 +          }
 +
 +          for (FileRef path : pathsToWaitFor) {
 +            if (fileScanReferenceCounts.get(path) > 0)
 +              inUse.add(path);
 +          }
 +
 +          if (blockNewScans) {
 +            reservationsBlocked = false;
 +            Tablet.this.notifyAll();
 +          }
 +
 +        }
 +      } finally {
 +        waitForScans.stop();
 +      }
 +      return inUse;
 +    }
 +
 +    public void importMapFiles(long tid, Map<FileRef,DataFileValue> pathsString, boolean setTime) throws IOException {
 +
 +      String bulkDir = null;
 +
 +      Map<FileRef,DataFileValue> paths = new HashMap<FileRef,DataFileValue>();
 +      for (Entry<FileRef,DataFileValue> entry : pathsString.entrySet())
 +        paths.put(entry.getKey(), entry.getValue());
 +
 +      for (FileRef tpath : paths.keySet()) {
 +
 +        boolean inTheRightDirectory = false;
 +        Path parent = tpath.path().getParent().getParent();
 +        for (String tablesDir : ServerConstants.getTablesDirs()) {
 +          if (parent.equals(new Path(tablesDir, extent.getTableId().toString()))) {
 +            inTheRightDirectory = true;
 +            break;
 +          }
 +        }
 +        if (!inTheRightDirectory) {
 +          throw new IOException("Data file " + tpath + " not in table dirs");
 +        }
 +
 +        if (bulkDir == null)
 +          bulkDir = tpath.path().getParent().toString();
 +        else if (!bulkDir.equals(tpath.path().getParent().toString()))
 +          throw new IllegalArgumentException("bulk files in different dirs " + bulkDir + " " + tpath);
 +
 +      }
 +
 +      if (extent.isRootTablet()) {
 +        throw new IllegalArgumentException("Can not import files to root tablet");
 +      }
 +
 +      synchronized (bulkFileImportLock) {
 +        Credentials creds = SystemCredentials.get();
 +        Connector conn;
 +        try {
 +          conn = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
 +        } catch (Exception ex) {
 +          throw new IOException(ex);
 +        }
 +        // Remove any bulk files we've previously loaded and compacted away
 +        List<FileRef> files = MetadataTableUtil.getBulkFilesLoaded(conn, extent, tid);
 +
 +        for (FileRef file : files)
 +          if (paths.keySet().remove(file))
 +            log.debug("Ignoring request to re-import a file already imported: " + extent + ": " + file);
 +
 +        if (paths.size() > 0) {
 +          long bulkTime = Long.MIN_VALUE;
 +          if (setTime) {
 +            for (DataFileValue dfv : paths.values()) {
 +              long nextTime = tabletTime.getAndUpdateTime();
 +              if (nextTime < bulkTime)
 +                throw new IllegalStateException("Time went backwards unexpectedly " + nextTime + " " + bulkTime);
 +              bulkTime = nextTime;
 +              dfv.setTime(bulkTime);
 +            }
 +          }
 +
 +          synchronized (timeLock) {
 +            if (bulkTime > persistedTime)
 +              persistedTime = bulkTime;
 +
 +            MetadataTableUtil.updateTabletDataFile(tid, extent, paths, tabletTime.getMetadataValue(persistedTime), creds, tabletServer.getLock());
 +          }
 +        }
 +      }
 +
 +      synchronized (Tablet.this) {
 +        for (Entry<FileRef,DataFileValue> tpath : paths.entrySet()) {
 +          if (datafileSizes.containsKey(tpath.getKey())) {
 +            log.error("Adding file that is already in set " + tpath.getKey());
 +          }
 +          datafileSizes.put(tpath.getKey(), tpath.getValue());
 +
 +        }
 +
 +        tabletResources.importedMapFiles();
 +
 +        computeNumEntries();
 +      }
 +
 +      for (Entry<FileRef,DataFileValue> entry : paths.entrySet()) {
 +        log.log(TLevel.TABLET_HIST, extent + " import " + entry.getKey() + " " + entry.getValue());
 +      }
 +    }
 +
 +    FileRef reserveMergingMinorCompactionFile() {
 +      if (mergingMinorCompactionFile != null)
 +        throw new IllegalStateException("Tried to reserve merging minor compaction file when already reserved  : " + mergingMinorCompactionFile);
 +
 +      if (extent.isRootTablet())
 +        return null;
 +
 +      int maxFiles = acuTableConf.getMaxFilesPerTablet();
 +
 +      // when a major compaction is running and we are at max files, write out
 +      // one extra file... want to avoid the case where major compaction is
 +      // compacting everything except for the largest file, and therefore the
 +      // largest file is returned for merging.. the following check mostly
 +      // avoids this case, except for the case where major compactions fail or
 +      // are canceled
 +      if (majorCompactingFiles.size() > 0 && datafileSizes.size() == maxFiles)
 +        return null;
 +
 +      if (datafileSizes.size() >= maxFiles) {
 +        // find the smallest file
 +
 +        long min = Long.MAX_VALUE;
 +        FileRef minName = null;
 +
 +        for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
 +          if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey())) {
 +            min = entry.getValue().getSize();
 +            minName = entry.getKey();
 +          }
 +        }
 +
 +        if (minName == null)
 +          return null;
 +
 +        mergingMinorCompactionFile = minName;
 +        return minName;
 +      }
 +
 +      return null;
 +    }
 +
 +    void unreserveMergingMinorCompactionFile(FileRef file) {
 +      if ((file == null && mergingMinorCompactionFile != null) || (file != null && mergingMinorCompactionFile == null)
 +          || (file != null && mergingMinorCompactionFile != null && !file.equals(mergingMinorCompactionFile)))
 +        throw new IllegalStateException("Disagreement " + file + " " + mergingMinorCompactionFile);
 +
 +      mergingMinorCompactionFile = null;
 +    }
 +
 +    void bringMinorCompactionOnline(FileRef tmpDatafile, FileRef newDatafile, FileRef absMergeFile, DataFileValue dfv, CommitSession commitSession, long flushId)
 +        throws IOException {
 +
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      if (extent.isRootTablet()) {
 +        try {
 +          if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
 +            throw new IllegalStateException();
 +          }
 +        } catch (Exception e) {
 +          throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
 +        }
 +      }
 +
 +      // rename before putting in metadata table, so files in metadata table should
 +      // always exist
 +      do {
 +        try {
 +          if (dfv.getNumEntries() == 0) {
 +            fs.deleteRecursively(tmpDatafile.path());
 +          } else {
 +            if (fs.exists(newDatafile.path())) {
 +              log.warn("Target map file already exist " + newDatafile);
 +              fs.deleteRecursively(newDatafile.path());
 +            }
 +
 +            rename(fs, tmpDatafile.path(), newDatafile.path());
 +          }
 +          break;
 +        } catch (IOException ioe) {
 +          log.warn("Tablet " + extent + " failed to rename " + newDatafile + " after MinC, will retry in 60 secs...", ioe);
 +          UtilWaitThread.sleep(60 * 1000);
 +        }
 +      } while (true);
 +
 +      long t1, t2;
 +
 +      // the code below always assumes merged files are in use by scans... this must be done
 +      // because the in memory list of files is not updated until after the metadata table
 +      // therefore the file is available to scans until memory is updated, but want to ensure
 +      // the file is not available for garbage collection... if memory were updated
 +      // before this point (like major compactions do), then the following code could wait
 +      // for scans to finish like major compactions do.... used to wait for scans to finish
 +      // here, but that was incorrect because a scan could start after waiting but before
 +      // memory was updated... assuming the file is always in use by scans leads to
 +      // one uneeded metadata update when it was not actually in use
 +      Set<FileRef> filesInUseByScans = Collections.emptySet();
 +      if (absMergeFile != null)
 +        filesInUseByScans = Collections.singleton(absMergeFile);
 +
 +      // very important to write delete entries outside of log lock, because
 +      // this metadata write does not go up... it goes sideways or to itself
 +      if (absMergeFile != null)
 +        MetadataTableUtil.addDeleteEntries(extent, Collections.singleton(absMergeFile), SystemCredentials.get());
 +
 +      Set<String> unusedWalLogs = beginClearingUnusedLogs();
 +      try {
 +        // the order of writing to metadata and walog is important in the face of machine/process failures
 +        // need to write to metadata before writing to walog, when things are done in the reverse order
 +        // data could be lost... the minor compaction start even should be written before the following metadata
 +        // write is made
 +
 +        synchronized (timeLock) {
 +          if (commitSession.getMaxCommittedTime() > persistedTime)
 +            persistedTime = commitSession.getMaxCommittedTime();
 +
 +          String time = tabletTime.getMetadataValue(persistedTime);
 +          MasterMetadataUtil.updateTabletDataFile(extent, newDatafile, absMergeFile, dfv, time, SystemCredentials.get(), filesInUseByScans,
 +              tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, lastLocation, flushId);
 +        }
 +
 +      } finally {
 +        finishClearingUnusedLogs();
 +      }
 +
 +      do {
 +        try {
 +          // the purpose of making this update use the new commit session, instead of the old one passed in,
 +          // is because the new one will reference the logs used by current memory...
 +
 +          tabletServer.minorCompactionFinished(tabletMemory.getCommitSession(), newDatafile.toString(), commitSession.getWALogSeq() + 2);
 +          break;
 +        } catch (IOException e) {
 +          log.error("Failed to write to write-ahead log " + e.getMessage() + " will retry", e);
 +          UtilWaitThread.sleep(1 * 1000);
 +        }
 +      } while (true);
 +
 +      synchronized (Tablet.this) {
 +        lastLocation = null;
 +
 +        t1 = System.currentTimeMillis();
 +        if (datafileSizes.containsKey(newDatafile)) {
 +          log.error("Adding file that is already in set " + newDatafile);
 +        }
 +
 +        if (dfv.getNumEntries() > 0) {
 +          datafileSizes.put(newDatafile, dfv);
 +        }
 +
 +        if (absMergeFile != null) {
 +          datafileSizes.remove(absMergeFile);
 +        }
 +
 +        unreserveMergingMinorCompactionFile(absMergeFile);
 +
 +        dataSourceDeletions.incrementAndGet();
 +        tabletMemory.finishedMinC();
 +
 +        lastFlushID = flushId;
 +
 +        computeNumEntries();
 +        t2 = System.currentTimeMillis();
 +      }
 +
 +      // must do this after list of files in memory is updated above
 +      removeFilesAfterScan(filesInUseByScans);
 +
 +      if (absMergeFile != null)
 +        log.log(TLevel.TABLET_HIST, extent + " MinC [" + absMergeFile + ",memory] -> " + newDatafile);
 +      else
 +        log.log(TLevel.TABLET_HIST, extent + " MinC [memory] -> " + newDatafile);
 +      log.debug(String.format("MinC finish lock %.2f secs %s", (t2 - t1) / 1000.0, getExtent().toString()));
 +      if (dfv.getSize() > acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD)) {
 +        log.debug(String.format("Minor Compaction wrote out file larger than split threshold.  split threshold = %,d  file size = %,d",
 +            acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD), dfv.getSize()));
 +      }
 +
 +    }
 +
 +    public void reserveMajorCompactingFiles(Collection<FileRef> files) {
 +      if (majorCompactingFiles.size() != 0)
 +        throw new IllegalStateException("Major compacting files not empty " + majorCompactingFiles);
 +
 +      if (mergingMinorCompactionFile != null && files.contains(mergingMinorCompactionFile))
 +        throw new IllegalStateException("Major compaction tried to resrve file in use by minor compaction " + mergingMinorCompactionFile);
 +
 +      majorCompactingFiles.addAll(files);
 +    }
 +
 +    public void clearMajorCompactingFile() {
 +      majorCompactingFiles.clear();
 +    }
 +
 +    void bringMajorCompactionOnline(Set<FileRef> oldDatafiles, FileRef tmpDatafile, FileRef newDatafile, Long compactionId, DataFileValue dfv)
 +        throws IOException {
 +      long t1, t2;
 +
 +      if (!extent.isRootTablet()) {
 +
 +        if (fs.exists(newDatafile.path())) {
 +          log.error("Target map file already exist " + newDatafile, new Exception());
 +          throw new IllegalStateException("Target map file already exist " + newDatafile);
 +        }
 +
 +        // rename before putting in metadata table, so files in metadata table should
 +        // always exist
 +        rename(fs, tmpDatafile.path(), newDatafile.path());
 +
 +        if (dfv.getNumEntries() == 0) {
 +          fs.deleteRecursively(newDatafile.path());
 +        }
 +      }
 +
 +      TServerInstance lastLocation = null;
 +      synchronized (Tablet.this) {
 +
 +        t1 = System.currentTimeMillis();
 +
 +        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +        dataSourceDeletions.incrementAndGet();
 +
 +        if (extent.isRootTablet()) {
 +
 +          waitForScansToFinish(oldDatafiles, true, Long.MAX_VALUE);
 +
 +          try {
 +            if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
 +              throw new IllegalStateException();
 +            }
 +          } catch (Exception e) {
 +            throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
 +          }
 +
 +          // mark files as ready for deletion, but
 +          // do not delete them until we successfully
 +          // rename the compacted map file, in case
 +          // the system goes down
 +
 +          RootFiles.replaceFiles(acuTableConf, fs, location, oldDatafiles, tmpDatafile, newDatafile);
 +        }
 +
 +        // atomically remove old files and add new file
 +        for (FileRef oldDatafile : oldDatafiles) {
 +          if (!datafileSizes.containsKey(oldDatafile)) {
 +            log.error("file does not exist in set " + oldDatafile);
 +          }
 +          datafileSizes.remove(oldDatafile);
 +          majorCompactingFiles.remove(oldDatafile);
 +        }
 +
 +        if (datafileSizes.containsKey(newDatafile)) {
 +          log.error("Adding file that is already in set " + newDatafile);
 +        }
 +
 +        if (dfv.getNumEntries() > 0) {
 +          datafileSizes.put(newDatafile, dfv);
 +        }
 +
 +        // could be used by a follow on compaction in a multipass compaction
 +        majorCompactingFiles.add(newDatafile);
 +
 +        computeNumEntries();
 +
 +        lastLocation = Tablet.this.lastLocation;
 +        Tablet.this.lastLocation = null;
 +
 +        if (compactionId != null)
 +          lastCompactID = compactionId;
 +
 +        t2 = System.currentTimeMillis();
 +      }
 +
 +      if (!extent.isRootTablet()) {
 +        Set<FileRef> filesInUseByScans = waitForScansToFinish(oldDatafiles, false, 10000);
 +        if (filesInUseByScans.size() > 0)
 +          log.debug("Adding scan refs to metadata " + extent + " " + filesInUseByScans);
 +        MasterMetadataUtil.replaceDatafiles(extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, SystemCredentials.get(),
 +            tabletServer.getClientAddressString(), lastLocation, tabletServer.getLock());
 +        removeFilesAfterScan(filesInUseByScans);
 +      }
 +
 +      log.debug(String.format("MajC finish lock %.2f secs", (t2 - t1) / 1000.0));
 +      log.log(TLevel.TABLET_HIST, extent + " MajC " + oldDatafiles + " --> " + newDatafile);
 +    }
 +
 +    public SortedMap<FileRef,DataFileValue> getDatafileSizes() {
 +      synchronized (Tablet.this) {
 +        TreeMap<FileRef,DataFileValue> copy = new TreeMap<FileRef,DataFileValue>(datafileSizes);
 +        return Collections.unmodifiableSortedMap(copy);
 +      }
 +    }
 +
 +    public Set<FileRef> getFiles() {
 +      synchronized (Tablet.this) {
 +        HashSet<FileRef> files = new HashSet<FileRef>(datafileSizes.keySet());
 +        return Collections.unmodifiableSet(files);
 +      }
 +    }
 +
 +  }
 +
 +  public Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
 +      throws IOException {
 +    this(tabletServer, location, extent, trm, CachedConfiguration.getInstance(), tabletsKeyValues);
 +    splitCreationTime = 0;
 +  }
 +
 +  public Tablet(KeyExtent extent, TabletServer tabletServer, TabletResourceManager trm, SplitInfo info) throws IOException {
 +    this(tabletServer, new Text(info.dir), extent, trm, CachedConfiguration.getInstance(), info.datafiles, info.time, info.initFlushID, info.initCompactID, info.lastLocation);
 +    splitCreationTime = System.currentTimeMillis();
 +  }
 +
 +  /**
 +   * Only visibile for testing
 +   */
 +  @VisibleForTesting
 +  protected Tablet(TabletTime tabletTime, String tabletDirectory, int logId, Path location, DatafileManager datafileManager) {
 +    this.tabletTime = tabletTime;
 +    this.tabletDirectory = tabletDirectory;
 +    this.logId = logId;
 +    this.location = location;
 +    this.datafileManager = datafileManager;
 +  }
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +    this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), tabletsKeyValues);
 +  }
 +
 +  static private final List<LogEntry> EMPTY = Collections.emptyList();
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
 +      SortedMap<FileRef,DataFileValue> datafiles, String time, long initFlushID, long initCompactID, TServerInstance last) throws IOException {
 +    this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), EMPTY, datafiles, time, last, new HashSet<FileRef>(), initFlushID, initCompactID);
 +  }
 +
 +  private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    SortedMap<Key,Value> entries;
 +
 +    if (extent.isRootTablet()) {
 +      return null;
 +    } else {
 +      entries = new TreeMap<Key,Value>();
 +      Text rowName = extent.getMetadataEntry();
 +      for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +        if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
 +          entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
 +        }
 +      }
 +    }
 +
 +    // log.debug("extent : "+extent+"   entries : "+entries);
 +
 +    if (entries.size() == 1)
 +      return entries.values().iterator().next().toString();
 +    return null;
 +  }
 +
 +  private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +
 +    TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
 +
 +    if (extent.isRootTablet()) { // the meta0 tablet
 +      Path location = new Path(MetadataTableUtil.getRootTabletDir());
 +
 +      // cleanUpFiles() has special handling for delete. files
 +      FileStatus[] files = fs.listStatus(location);
 +      Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
 +      for (String good : goodPaths) {
 +        Path path = new Path(good);
 +        String filename = path.getName();
 +        FileRef ref = new FileRef(location.toString() + "/" + filename, path);
 +        DataFileValue dfv = new DataFileValue(0, 0);
 +        datafiles.put(ref, dfv);
 +      }
 +    } else {
 +
 +      Text rowName = extent.getMetadataEntry();
 +
 +      String tableId = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
 +      ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), tableId, Authorizations.EMPTY);
 +
 +      // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
 +      // reduced batch size to improve performance
 +      // changed here after endKeys were implemented from 10 to 1000
 +      mdScanner.setBatchSize(1000);
 +
 +      // leave these in, again, now using endKey for safety
 +      mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +
 +      mdScanner.setRange(new Range(rowName));
 +
 +      for (Entry<Key,Value> entry : mdScanner) {
 +
 +        if (entry.getKey().compareRow(rowName) != 0) {
 +          break;
 +        }
 +
 +        FileRef ref = new FileRef(fs, entry.getKey());
 +        datafiles.put(ref, new DataFileValue(entry.getValue().get()));
 +      }
 +    }
 +    return datafiles;
 +  }
 +
 +  private static List<LogEntry> lookupLogEntries(KeyExtent ke, SortedMap<Key,Value> tabletsKeyValues) {
 +    List<LogEntry> logEntries = new ArrayList<LogEntry>();
 +
 +    if (ke.isMeta()) {
 +      try {
 +        logEntries = MetadataTableUtil.getLogEntries(SystemCredentials.get(), ke);
 +      } catch (Exception ex) {
 +        throw new RuntimeException("Unable to read tablet log entries", ex);
 +      }
 +    } else {
 +      log.debug("Looking at metadata " + tabletsKeyValues);
 +      Text row = ke.getMetadataEntry();
 +      for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +        Key key = entry.getKey();
 +        if (key.getRow().equals(row)) {
 +          if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
 +            logEntries.add(LogEntry.fromKeyValue(key, entry.getValue()));
 +          }
 +        }
 +      }
 +    }
 +
 +    log.debug("got " + logEntries + " for logs for " + ke);
 +    return logEntries;
 +  }
 +
 +  private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
 +    HashSet<FileRef> scanFiles = new HashSet<FileRef>();
 +
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
 +        scanFiles.add(new FileRef(fs, key));
 +      }
 +    }
 +
 +    return scanFiles;
 +  }
 +
 +  private static long lookupFlushID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +        return Long.parseLong(entry.getValue().toString());
 +    }
 +
 +    return -1;
 +  }
 +
 +  private static long lookupCompactID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    Text row = extent.getMetadataEntry();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      Key key = entry.getKey();
 +      if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
 +        return Long.parseLong(entry.getValue().toString());
 +    }
 +
 +    return -1;
 +  }
 +
 +  private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, VolumeManager fs,
 +      SortedMap<Key,Value> tabletsKeyValues) throws IOException {
 +    this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(), fs,
 +        extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent, tabletsKeyValues),
 +        lookupScanFiles(extent, tabletsKeyValues, fs), lookupFlushID(extent, tabletsKeyValues), lookupCompactID(extent, tabletsKeyValues));
 +  }
 +
 +  private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
 +        return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
 +      }
 +    }
 +    return null;
 +  }
 +
 +  /**
 +   * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
 +   */
 +  private Tablet(final TabletServer tabletServer, final Text location, final KeyExtent extent, final TabletResourceManager trm, final Configuration conf,
 +      final VolumeManager fs, final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time,
 +      final TServerInstance lastLocation, Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
 +
 +    TabletFiles tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getLock(), fs, extent, new TabletFiles(location.toString(), rawLogEntries,
 +        rawDatafiles));
 +
 +    Path locationPath;
 +
 +    if (tabletPaths.dir.contains(":")) {
 +      locationPath = new Path(tabletPaths.dir.toString());
 +    } else {
 +      locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString());
 +    }
 +
 +    final List<LogEntry> logEntries = tabletPaths.logEntries;
 +    final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
 +
 +    this.location = locationPath;
 +    this.lastLocation = lastLocation;
 +    this.tabletDirectory = tabletPaths.dir;
 +    this.conf = conf;
 +    this.acuTableConf = tabletServer.getTableConfiguration(extent);
 +
 +    this.fs = fs;
 +    this.extent = extent;
 +    this.tabletResources = trm;
 +
 +    this.lastFlushID = initFlushID;
 +    this.lastCompactID = initCompactID;
 +
 +    if (extent.isRootTablet()) {
 +      long rtime = Long.MIN_VALUE;
 +      for (FileRef ref : datafiles.keySet()) {
 +        Path path = ref.path();
 +        FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
 +        FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
 +        long maxTime = -1;
 +        try {
 +
 +          while (reader.hasTop()) {
 +            maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
 +            reader.next();
 +          }
 +
 +        } finally {
 +          reader.close();
 +        }
 +
 +        if (maxTime > rtime) {
 +          time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
 +          rtime = maxTime;
 +        }
 +      }
 +    }
 +    if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
 +      // recovery... old root tablet has no data, so time doesn't matter:
 +      time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
 +    }
 +
 +    this.tabletServer = tabletServer;
 +    this.logId = tabletServer.createLogId(extent);
 +
 +    this.timer = new TabletStatsKeeper();
 +
 +    setupDefaultSecurityLabels(extent);
 +
 +    tabletMemory = new TabletMemory();
 +    tabletTime = TabletTime.getInstance(time);
 +    persistedTime = tabletTime.getTime();
 +
 +    acuTableConf.addObserver(configObserver = new ConfigurationObserver() {
 +
 +      private void reloadConstraints() {
 +        constraintChecker.set(new ConstraintChecker(acuTableConf));
 +      }
 +
 +      @Override
 +      public void propertiesChanged() {
 +        reloadConstraints();
 +
 +        try {
 +          setupDefaultSecurityLabels(extent);
 +        } catch (Exception e) {
 +          log.error("Failed to reload default security labels for extent: " + extent.toString());
 +        }
 +      }
 +
 +      @Override
 +      public void propertyChanged(String prop) {
 +        if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
 +          reloadConstraints();
 +        else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
 +          try {
 +            log.info("Default security labels changed for extent: " + extent.toString());
 +            setupDefaultSecurityLabels(extent);
 +          } catch (Exception e) {
 +            log.error("Failed to reload default security labels for extent: " + extent.toString());
 +          }
 +        }
 +
 +      }
 +
 +      @Override
 +      public void sessionExpired() {
 +        log.debug("Session expired, no longer updating per table props...");
 +      }
 +
 +    });
 +
 +    acuTableConf.getNamespaceConfiguration().addObserver(configObserver);
 +
 +    // Force a load of any per-table properties
 +    configObserver.propertiesChanged();
 +
 +    tabletResources.setTablet(this, acuTableConf);
 +    if (!logEntries.isEmpty()) {
 +      log.info("Starting Write-Ahead Log recovery for " + this.extent);
 +      // count[0] = entries used on tablet
 +      // count[1] = track max time from walog entries wihtout timestamps
 +      final long[] count = new long[2];
 +      final CommitSession commitSession = tabletMemory.getCommitSession();
 +      count[1] = Long.MIN_VALUE;
 +      try {
 +        Set<String> absPaths = new HashSet<String>();
 +        for (FileRef ref : datafiles.keySet())
 +          absPaths.add(ref.path().toString());
 +
 +        tabletServer.recover(this.tabletServer.getFileSystem(), this, logEntries, absPaths, new MutationReceiver() {
 +          @Override
 +          public void receive(Mutation m) {
 +            // LogReader.printMutation(m);
 +            Collection<ColumnUpdate> muts = m.getUpdates();
 +            for (ColumnUpdate columnUpdate : muts) {
 +              if (!columnUpdate.hasTimestamp()) {
 +                // if it is not a user set timestamp, it must have been set
 +                // by the system
 +                count[1] = Math.max(count[1], columnUpdate.getTimestamp());
 +              }
 +            }
 +            tabletMemory.mutate(commitSession, Collections.singletonList(m));
 +            count[0]++;
 +          }
 +        });
 +
 +        if (count[1] != Long.MIN_VALUE) {
 +          tabletTime.useMaxTimeFromWALog(count[1]);
 +        }
 +        commitSession.updateMaxCommittedTime(tabletTime.getTime());
 +
 +        if (count[0] == 0) {
 +          log.debug("No replayed mutations applied, removing unused entries for " + extent);
 +          MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
 +          logEntries.clear();
 +        }
 +
 +      } catch (Throwable t) {
 +        if (acuTableConf.getBoolean(Property.TABLE_FAILURES_IGNORE)) {
 +          log.warn("Error recovering from log files: ", t);
 +        } else {
 +          throw new RuntimeException(t);
 +        }
 +      }
 +      // make some closed references that represent the recovered logs
 +      currentLogs = new HashSet<DfsLogger>();
 +      for (LogEntry logEntry : logEntries) {
 +        for (String log : logEntry.logSet) {
 +          currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), log, logEntry.getColumnQualifier().toString()));
 +        }
 +      }
 +
 +      log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries()
 +          + " entries created)");
 +    }
 +
 +    String contextName = acuTableConf.get(Property.TABLE_CLASSPATH);
 +    if (contextName != null && !contextName.equals("")) {
 +      // initialize context classloader, instead of possibly waiting for it to initialize for a scan
 +      // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292
 +      AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
 +    }
 +
 +    // do this last after tablet is completely setup because it
 +    // could cause major compaction to start
 +    datafileManager = new DatafileManager(datafiles);
 +
 +    computeNumEntries();
 +
 +    datafileManager.removeFilesAfterScan(scanFiles);
 +
 +    // look for hints of a failure on the previous tablet server
 +    if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
 +      // look for any temp files hanging around
 +      removeOldTemporaryFiles();
 +    }
 +
 +    log.log(TLevel.TABLET_HIST, extent + " opened");
 +  }
 +
 +  private void removeOldTemporaryFiles() {
 +    // remove any temporary files created by a previous tablet server
 +    try {
 +      for (FileStatus tmp : fs.globStatus(new Path(location, "*_tmp"))) {
 +        try {
 +          log.debug("Removing old temp file " + tmp.getPath());
 +          fs.delete(tmp.getPath());
 +        } catch (IOException ex) {
 +          log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
 +        }
 +      }
 +    } catch (IOException ex) {
 +      log.error("Error scanning for old temp files in " + location);
 +    }
 +  }
 +
 +  private void setupDefaultSecurityLabels(KeyExtent extent) {
 +    if (extent.isMeta()) {
 +      defaultSecurityLabel = new byte[0];
 +    } else {
 +      try {
 +        ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
 +        this.defaultSecurityLabel = cv.getExpression();
 +      } catch (Exception e) {
 +        log.error(e, e);
 +        this.defaultSecurityLabel = new byte[0];
 +      }
 +    }
 +  }
 +
 +  public static class KVEntry extends KeyValue {
 +    private static final long serialVersionUID = 1L;
 +
 +    public KVEntry(Key k, Value v) {
 +      super(new Key(k), Arrays.copyOf(v.get(), v.get().length));
 +    }
 +
 +    int numBytes() {
 +      return getKey().getSize() + getValue().get().length;
 +    }
 +
 +    int estimateMemoryUsed() {
 +      return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object
 +    }
 +  }
 +
 +  private LookupResult lookup(SortedKeyValueIterator<Key,Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, ArrayList<KVEntry> results,
 +      long maxResultsSize) throws IOException {
 +
 +    LookupResult lookupResult = new LookupResult();
 +
 +    boolean exceededMemoryUsage = false;
 +    boolean tabletClosed = false;
 +
 +    Set<ByteSequence> cfset = null;
 +    if (columnSet.size() > 0)
 +      cfset = LocalityGroupUtil.families(columnSet);
 +
 +    for (Range range : ranges) {
 +
 +      if (exceededMemoryUsage || tabletClosed) {
 +        lookupResult.unfinishedRanges.add(range);
 +        continue;
 +      }
 +
 +      int entriesAdded = 0;
 +
 +      try {
 +        if (cfset != null)
 +          mmfi.seek(range, cfset, true);
 +        else
 +          mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
 +
 +        while (mmfi.hasTop()) {
 +          Key key = mmfi.getTopKey();
 +
 +          KVEntry kve = new KVEntry(key, mmfi.getTopValue());
 +          results.add(kve);
 +          entriesAdded++;
 +          lookupResult.bytesAdded += kve.estimateMemoryUsed();
 +          lookupResult.dataSize += kve.numBytes();
 +
 +          exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
 +
 +          if (exceededMemoryUsage) {
 +            addUnfinishedRange(lookupResult, range, key, false);
 +            break;
 +          }
 +
 +          mmfi.next();
 +        }
 +
 +      } catch (TooManyFilesException tmfe) {
 +        // treat this as a closed tablet, and let the client retry
 +        log.warn("Tablet " + getExtent() + " has too many files, batch lookup can not run");
 +        handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +        tabletClosed = true;
 +      } catch (IOException ioe) {
 +        if (shutdownInProgress()) {
 +          // assume HDFS shutdown hook caused this exception
 +          log.debug("IOException while shutdown in progress ", ioe);
 +          handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +          tabletClosed = true;
 +        } else {
 +          throw ioe;
 +        }
 +      } catch (IterationInterruptedException iie) {
 +        if (isClosed()) {
 +          handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +          tabletClosed = true;
 +        } else {
 +          throw iie;
 +        }
 +      } catch (TabletClosedException tce) {
 +        handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
 +        tabletClosed = true;
 +      }
 +
 +    }
 +
 +    return lookupResult;
 +  }
 +
 +  private void handleTabletClosedDuringScan(ArrayList<KVEntry> results, LookupResult lookupResult, boolean exceededMemoryUsage, Range range, int entriesAdded) {
 +    if (exceededMemoryUsage)
 +      throw new IllegalStateException("tablet should not exceed memory usage or close, not both");
 +
 +    if (entriesAdded > 0)
 +      addUnfinishedRange(lookupResult, range, results.get(results.size() - 1).getKey(), false);
 +    else
 +      lookupResult.unfinishedRanges.add(range);
 +
 +    lookupResult.closed = true;
 +  }
 +
 +  private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) {
 +    if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) {
 +      Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive());
 +      lookupResult.unfinishedRanges.add(nlur);
 +    }
 +  }
 +
 +  public static interface KVReceiver {
 +    void receive(List<KVEntry> matches) throws IOException;
 +  }
 +
 +  class LookupResult {
 +    List<Range> unfinishedRanges = new ArrayList<Range>();
 +    long bytesAdded = 0;
 +    long dataSize = 0;
 +    boolean closed = false;
 +  }
 +
 +  public LookupResult lookup(List<Range> ranges, HashSet<Column> columns, Authorizations authorizations, ArrayList<KVEntry> results, long maxResultSize,
 +      List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag) throws IOException {
 +
 +    if (ranges.size() == 0) {
 +      return new LookupResult();
 +    }
 +
 +    ranges = Range.mergeOverlapping(ranges);
 +    Collections.sort(ranges);
 +
 +    Range tabletRange = extent.toDataRange();
 +    for (Range range : ranges) {
 +      // do a test to see if this range falls within the tablet, if it does not
 +      // then clip will throw an exception
 +      tabletRange.clip(range);
 +    }
 +
 +    ScanDataSource dataSource = new ScanDataSource(authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
 +
 +    LookupResult result = null;
 +
 +    try {
 +      SortedKeyValueIterator<Key,Value> iter = new SourceSwitchingIterator(dataSource);
 +      result = lookup(iter, ranges, columns, results, maxResultSize);
 +      return result;
 +    } catch (IOException ioe) {
 +      dataSource.close(true);
 +      throw ioe;
 +    } finally {
 +      // code in finally block because always want
 +      // to return mapfiles, even when exception is thrown
 +      dataSource.close(false);
 +
 +      synchronized (this) {
 +        queryCount += results.size();
 +        if (result != null)
 +          queryBytes += result.dataSize;
 +      }
 +    }
 +  }
 +
 +  private Batch nextBatch(SortedKeyValueIterator<Key,Value> iter, Range range, int num, Set<Column> columns) throws IOException {
 +
 +    // log.info("In nextBatch..");
 +
 +    List<KVEntry> results = new ArrayList<KVEntry>();
 +    Key key = null;
 +
 +    Value value;
 +    long resultSize = 0L;
 +    long resultBytes = 0L;
 +
 +    long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
 +
 +    if (columns.size() == 0) {
 +      iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
 +    } else {
 +      iter.seek(range, LocalityGroupUtil.families(columns), true);
 +    }
 +
 +    Key continueKey = null;
 +    boolean skipContinueKey = false;
 +
 +    boolean endOfTabletReached = false;
 +    while (iter.hasTop()) {
 +
 +      value = iter.getTopValue();
 +      key = iter.getTopKey();
 +
 +      KVEntry kvEntry = new KVEntry(key, value); // copies key and value
 +      results.add(kvEntry);
 +      resultSize += kvEntry.estimateMemoryUsed();
 +      resultBytes += kvEntry.numBytes();
 +
 +      if (resultSize >= maxResultsSize || results.size() >= num) {
 +        continueKey = new Key(key);
 +        skipContinueKey = true;
 +        break;
 +      }
 +
 +      iter.next();
 +    }
 +
 +    if (iter.hasTop() == false) {
 +      endOfTabletReached = true;
 +    }
 +
 +    Batch retBatch = new Batch();
 +    retBatch.numBytes = resultBytes;
 +
 +    if (!endOfTabletReached) {
 +      retBatch.continueKey = continueKey;
 +      retBatch.skipContinueKey = skipContinueKey;
 +    } else {
 +      retBatch.continueKey = null;
 +    }
 +
 +    if (endOfTabletReached && results.size() == 0)
 +      retBatch.results = null;
 +    else
 +      retBatch.results = results;
 +
 +    return retBatch;
 +  }
 +
 +  /**
 +   * Determine if a JVM shutdown is in progress.
 +   *
 +   */
 +  private boolean shutdownInProgress() {
 +    try {
 +      Runtime.getRuntime().removeShutdownHook(new Thread(new Runnable() {
 +        @Override
 +        public void run() {}
 +      }));
 +    } catch (IllegalStateException ise) {
 +      return true;
 +    }
 +
 +    return false;
 +  }
 +
 +  private class Batch {
 +    public boolean skipContinueKey;
 +    public List<KVEntry> results;
 +    public Key continueKey;
 +    public long numBytes;
 +  }
 +
 +  Scanner createScanner(Range range, int num, Set<Column> columns, Authorizations authorizations, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
 +      boolean isolated, AtomicBoolean interruptFlag) {
 +    // do a test to see if this range falls within the tablet, if it does not
 +    // then clip will throw an exception
 +    extent.toDataRange().clip(range);
 +
 +    ScanOptions opts = new ScanOptions(num, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag, isolated);
 +    return new Scanner(range, opts);
 +  }
 +
 +  class ScanBatch {
 +    boolean more;
 +    List<KVEntry> results;
 +
 +    ScanBatch(List<KVEntry> results, boolean more) {
 +      this.results = results;
 +      this.more = more;
 +    }
 +  }
 +
 +  class Scanner {
 +
 +    private ScanOptions options;
 +    private Range range;
 +    private SortedKeyValueIterator<Key,Value> isolatedIter;
 +    private ScanDataSource isolatedDataSource;
 +    private boolean sawException = false;
 +    private boolean scanClosed = false;
 +
 +    Scanner(Range range, ScanOptions options) {
 +      this.range = range;
 +      this.options = options;
 +    }
 +
 +    synchronized ScanBatch read() throws IOException, TabletClosedException {
 +
 +      if (sawException)
 +        throw new IllegalStateException("Tried to use scanner after exception occurred.");
 +
 +      if (scanClosed)
 +        throw new IllegalStateException("Tried to use scanner after it was closed.");
 +
 +      Batch results = null;
 +
 +      ScanDataSource dataSource;
 +
 +      if (options.isolated) {
 +        if (isolatedDataSource == null)
 +          isolatedDataSource = new ScanDataSource(options);
 +        dataSource = isolatedDataSource;
 +      } else {
 +        dataSource = new ScanDataSource(options);
 +      }
 +
 +      try {
 +
 +        SortedKeyValueIterator<Key,Value> iter;
 +
 +        if (options.isolated) {
 +          if (isolatedIter == null)
 +            isolatedIter = new SourceSwitchingIterator(dataSource, true);
 +          else
 +            isolatedDataSource.fileManager.reattach();
 +          iter = isolatedIter;
 +        } else {
 +          iter = new SourceSwitchingIterator(dataSource, false);
 +        }
 +
 +        results = nextBatch(iter, range, options.num, options.columnSet);
 +
 +        if (results.results == null) {
 +          range = null;
 +          return new ScanBatch(new ArrayList<Tablet.KVEntry>(), false);
 +        } else if (results.continueKey == null) {
 +          return new ScanBatch(results.results, false);
 +        } else {
 +          range = new Range(results.continueKey, !results.skipContinueKey, range.getEndKey(), range.isEndKeyInclusive());
 +          return new ScanBatch(results.results, true);
 +        }
 +
 +      } catch (IterationInterruptedException iie) {
 +        sawException = true;
 +        if (isClosed())
 +          throw new TabletClosedException(iie);
 +        else
 +          throw iie;
 +      } catch (IOException ioe) {
 +        if (shutdownInProgress()) {
 +          log.debug("IOException while shutdown in progress ", ioe);
 +          throw new TabletClosedException(ioe); // assume IOException was caused by execution of HDFS shutdown hook
 +        }
 +
 +        sawException = true;
 +        dataSource.close(true);
 +        throw ioe;
 +      } catch (RuntimeException re) {
 +        sawException = true;
 +        throw re;
 +      } finally {
 +        // code in finally block because always want
 +        // to return mapfiles, even when exception is thrown
 +        if (!options.isolated)
 +          dataSource.close(false);
 +        else if (dataSource.fileManager != null)
 +          dataSource.fileManager.detach();
 +
 +        synchronized (Tablet.this) {
 +          if (results != null && results.results != null) {
 +            long more = results.results.size();
 +            queryCount += more;
 +            queryBytes += results.numBytes;
 +          }
 +        }
 +      }
 +    }
 +
 +    // close and read are synchronized because can not call close on the data source while it is in use
 +    // this cloud lead to the case where file iterators that are in use by a thread are returned
 +    // to the pool... this would be bad
 +    void close() {
 +      options.interruptFlag.set(true);
 +      synchronized (this) {
 +        scanClosed = true;
 +        if (isolatedDataSource != null)
 +          isolatedDataSource.close(false);
 +      }
 +    }
 +  }
 +
 +  static class ScanOptions {
 +
 +    // scan options
 +    Authorizations authorizations;
 +    byte[] defaultLabels;
 +    Set<Column> columnSet;
 +    List<IterInfo> ssiList;
 +    Map<String,Map<String,String>> ssio;
 +    AtomicBoolean interruptFlag;
 +    int num;
 +    boolean isolated;
 +
 +    ScanOptions(int num, Authorizations authorizations, byte[] defaultLabels, Set<Column> columnSet, List<IterInfo> ssiList,
 +        Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag, boolean isolated) {
 +      this.num = num;
 +      this.authorizations = authorizations;
 +      this.defaultLabels = defaultLabels;
 +      this.columnSet = columnSet;
 +      this.ssiList = ssiList;
 +      this.ssio = ssio;
 +      this.interruptFlag = interruptFlag;
 +      this.isolated = isolated;
 +    }
 +
 +  }
 +
 +  class ScanDataSource implements DataSource {
 +
 +    // data source state
 +    private ScanFileManager fileManager;
 +    private SortedKeyValueIterator<Key,Value> iter;
 +    private long expectedDeletionCount;
 +    private List<MemoryIterator> memIters = null;
 +    private long fileReservationId;
 +    private AtomicBoolean interruptFlag;
 +    private StatsIterator statsIterator;
 +
 +    ScanOptions options;
 +
 +    ScanDataSource(Authorizations authorizations, byte[] defaultLabels, HashSet<Column> columnSet, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
 +        AtomicBoolean interruptFlag) {
 +      expectedDeletionCount = dataSourceDeletions.get();
 +      this.options = new ScanOptions(-1, authorizations, defaultLabels, columnSet, ssiList, ssio, interruptFlag, false);
 +      this.interruptFlag = interruptFlag;
 +    }
 +
 +    ScanDataSource(ScanOptions options) {
 +      expectedDeletionCount = dataSourceDeletions.get();
 +      this.options = options;
 +      this.interruptFlag = options.interruptFlag;
 +    }
 +
 +    @Override
 +    public DataSource getNewDataSource() {
 +      if (!isCurrent()) {
 +        // log.debug("Switching data sources during a scan");
 +        if (memIters != null) {
 +          tabletMemory.returnIterators(memIters);
 +          memIters = null;
 +          datafileManager.returnFilesForScan(fileReservationId);
 +          fileReservationId = -1;
 +        }
 +
 +        if (fileManager != null)
 +          fileManager.releaseOpenFiles(false);
 +
 +        expectedDeletionCount = dataSourceDeletions.get();
 +        iter = null;
 +
 +        return this;
 +      } else
 +        return this;
 +    }
 +
 +    @Override
 +    public boolean isCurrent() {
 +      return expectedDeletionCount == dataSourceDeletions.get();
 +    }
 +
 +    @Override
 +    public SortedKeyValueIterator<Key,Value> iterator() throws IOException {
 +      if (iter == null)
 +        iter = createIterator();
 +      return iter;
 +    }
 +
 +    private SortedKeyValueIterator<Key,Value> createIterator() throws IOException {
 +
 +      Map<FileRef,DataFileValue> files;
 +
 +      synchronized (Tablet.this) {
 +
 +        if (memIters != null)
 +          throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
 +
 +        if (Tablet.this.closed)
 +          throw new TabletClosedException();
 +
 +        if (interruptFlag.get())
 +          throw new IterationInterruptedException(extent.toString() + " " + interruptFlag.hashCode());
 +
 +        // only acquire the file manager when we know the tablet is open
 +        if (fileManager == null) {
 +          fileManager = tabletResources.newScanFileManager();
 +          activeScans.add(this);
 +        }
 +
 +        if (fileManager.getNumOpenFiles() != 0)
 +          throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
 +
 +        // set this before trying to get iterators in case
 +        // getIterators() throws an exception
 +        expectedDeletionCount = dataSourceDeletions.get();
 +
 +        memIters = tabletMemory.getIterators();
 +        Pair<Long,Map<FileRef,DataFileValue>> reservation = datafileManager.reserveFilesForScan();
 +        fileReservationId = reservation.getFirst();
 +        files = reservation.getSecond();
 +      }
 +
 +      Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, options.isolated);
 +
 +      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.size() + memIters.size());
 +
 +      iters.addAll(mapfiles);
 +      iters.addAll(memIters);
 +
 +      for (SortedKeyValueIterator<Key,Value> skvi : iters)
 +        ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
 +
 +      MultiIterator multiIter = new MultiIterator(iters, extent);
 +
 +      TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager, files);
 +
 +      statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, scannedCount);
 +
 +      DeletingIterator delIter = new DeletingIterator(statsIterator, false);
 +
 +      ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
 +
 +      ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, options.columnSet);
 +
 +      VisibilityFilter visFilter = new VisibilityFilter(colFilter, options.authorizations, options.defaultLabels);
 +
 +      return iterEnv.getTopLevelIterator(IteratorUtil
 +          .loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.ssiList, options.ssio, iterEnv));
 +    }
 +
 +    private void close(boolean sawErrors) {
 +
 +      if (memIters != null) {
 +        tabletMemory.returnIterators(memIters);
 +        memIters = null;
 +        datafileManager.returnFilesForScan(fileReservationId);
 +        fileReservationId = -1;
 +      }
 +
 +      synchronized (Tablet.this) {
 +        activeScans.remove(this);
 +        if (activeScans.size() == 0)
 +          Tablet.this.notifyAll();
 +      }
 +
 +      if (fileManager != null) {
 +        fileManager.releaseOpenFiles(sawErrors);
 +        fileManager = null;
 +      }
 +
 +      if (statsIterator != null) {
 +        statsIterator.report();
 +      }
 +
 +    }
 +
 +    public void interrupt() {
 +      interruptFlag.set(true);
 +    }
 +
 +    @Override
 +    public DataSource getDeepCopyDataSource(IteratorEnvironment env) {
 +      throw new UnsupportedOperationException();
 +    }
 +
 +    @Override
 +    public void setInterruptFlag(AtomicBoolean flag) {
 +      throw new UnsupportedOperationException();
 +    }
 +
 +  }
 +
 +  private DataFileValue minorCompact(Configuration conf, VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile,
 +      boolean hasQueueTime, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
 +    boolean failed = false;
 +    long start = System.currentTimeMillis();
 +    timer.incrementStatusMinor();
 +
 +    long count = 0;
 +
 +    try {
 +      Span span = Trace.start("write");
 +      CompactionStats stats;
 +      try {
 +        count = memTable.getNumEntries();
 +
 +        DataFileValue dfv = null;
 +        if (mergeFile != null)
 +          dfv = datafileManager.getDatafileSizes().get(mergeFile);
 +
 +        MinorCompactor compactor = new MinorCompactor(conf, fs, memTable, mergeFile, dfv, tmpDatafile, acuTableConf, extent, mincReason);
 +        stats = compactor.call();
 +      } finally {
 +        span.stop();
 +      }
 +      span = Trace.start("bringOnline");
 +      try {
 +        datafileManager.bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
 +            commitSession, flushId);
 +      } finally {
 +        span.stop();
 +      }
 +      return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
 +    } catch (Exception E) {
 +      failed = true;
 +      throw new RuntimeException(E);
 +    } catch (Error E) {
 +      // Weird errors like "OutOfMemoryError" when trying to create the thread for the compaction
 +      failed = true;
 +      throw new RuntimeException(E);
 +    } finally {
 +      try {
 +        tabletMemory.finalizeMinC();
 +      } catch (Throwable t) {
 +        log.error("Failed to free tablet memory", t);
 +      }
 +
 +      if (!failed) {
 +        lastMinorCompactionFinishTime = System.currentTimeMillis();
 +      }
 +      if (tabletServer.mincMetrics.isEnabled())
 +        tabletServer.mincMetrics.add(TabletServerMinCMetrics.minc, (lastMinorCompactionFinishTime - start));
 +      if (hasQueueTime) {
 +        timer.updateTime(Operation.MINOR, queued, start, count, failed);
 +        if (tabletServer.mincMetrics.isEnabled())
 +          tabletServer.mincMetrics.add(TabletServerMinCMetrics.queue, (start - queued));
 +      } else
 +        timer.updateTime(Operation.MINOR, start, count, failed);
 +    }
 +  }
 +
 +  private class MinorCompactionTask implements Runnable {
 +
 +    private long queued;
 +    private CommitSession commitSession;
 +    private DataFileValue stats;
 +    private FileRef mergeFile;
 +    private long flushId;
 +    private MinorCompactionReason mincReason;
 +
 +    MinorCompactionTask(FileRef mergeFile, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
 +      queued = System.currentTimeMillis();
 +      minorCompactionWaitingToStart = true;
 +      this.commitSession = commitSession;
 +      this.mergeFile = mergeFile;
 +      this.flushId = flushId;
 +      this.mincReason = mincReason;
 +    }
 +
 +    @Override
 +    public void run() {
 +      minorCompactionWaitingToStart = false;
 +      minorCompactionInProgress = true;
 +      Span minorCompaction = Trace.on("minorCompaction");
 +      try {
 +        FileRef newMapfileLocation = getNextMapFilename(mergeFile == null ? "F" : "M");
 +        FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp");
 +        Span span = Trace.start("waitForCommits");
 +        synchronized (Tablet.this) {
 +          commitSession.waitForCommitsToFinish();
 +        }
 +        span.stop();
 +        span = Trace.start("start");
 +        while (true) {
 +          try {
 +            // the purpose of the minor compaction start event is to keep track of the filename... in the case
 +            // where the metadata table write for the minor compaction finishes and the process dies before
 +            // writing the minor compaction finish event, then the start event+filename in metadata table will
 +            // prevent recovery of duplicate data... the minor compaction start event could be written at any time
 +            // before the metadata write for the minor compaction
 +            tabletServer.minorCompactionStarted(commitSession, commitSession.getWALogSeq() + 1, newMapfileLocation.path().toString());
 +            break;
 +          } catch (IOException e) {
 +            log.warn("Failed to write to write ahead log " + e.getMessage(), e);
 +          }
 +        }
 +        span.stop();
 +        span = Trace.start("compact");
 +        this.stats = minorCompact(conf, fs, tabletMemory.getMinCMemTable(), tmpFileRef, newMapfileLocation, mergeFile, true, queued, commitSession, flushId,
 +            mincReason);
 +        span.stop();
 +
 +        if (needsSplit()) {
 +          tabletServer.executeSplit(Tablet.this);
 +        } else {
 +          initiateMajorCompaction(MajorCompactionReason.NORMAL);
 +        }
 +      } catch (Throwable t) {
 +        log.error("Unknown error during minor compaction for extent: " + getExtent(), t);
 +        throw new RuntimeException(t);
 +      } finally {
 +        minorCompactionInProgress = false;
 +        minorCompaction.data("extent", extent.toString());
 +        minorCompaction.data("numEntries", Long.toString(this.stats.getNumEntries()));
 +        minorCompaction.data("size", Long.toString(this.stats.getSize()));
 +        minorCompaction.stop();
 +      }
 +    }
 +  }
 +
 +  private synchronized MinorCompactionTask prepareForMinC(long flushId, MinorCompactionReason mincReason) {
 +    CommitSession oldCommitSession = tabletMemory.prepareForMinC();
 +    otherLogs = currentLogs;
 +    currentLogs = new HashSet<DfsLogger>();
 +
 +    FileRef mergeFile = null;
 +    if (mincReason != MinorCompactionReason.RECOVERY) {
 +      mergeFile = datafileManager.reserveMergingMinorCompactionFile();
 +    }
 +
 +    return new MinorCompactionTask(mergeFile, oldCommitSession, flushId, mincReason);
 +
 +  }
 +
 +  void flush(long tableFlushID) {
 +    boolean updateMetadata = false;
 +    boolean initiateMinor = false;
 +
 +    try {
 +
 +      synchronized (this) {
 +
 +        // only want one thing at a time to update flush ID to ensure that metadata table and tablet in memory state are consistent
 +        if (updatingFlushID)
 +          return;
 +
 +        if (lastFlushID >= tableFlushID)
 +          return;
 +
 +        if (closing || closed || tabletMemory.memoryReservedForMinC())
 +          return;
 +
 +        if (tabletMemory.getMemTable().getNumEntries() == 0) {
 +          lastFlushID = tableFlushID;
 +          updatingFlushID = true;
 +          updateMetadata = true;
 +        } else
 +          initiateMinor = true;
 +      }
 +
 +      if (updateMetadata) {
 +        Credentials creds = SystemCredentials.get();
 +        // if multiple threads were allowed to update this outside of a sync block, then it would be
 +        // a race condition
 +        MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, tabletServer.getLock());
 +      } else if (initiateMinor)
 +        initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
 +
 +    } finally {
 +      if (updateMetadata) {
 +        synchronized (this) {
 +          updatingFlushID = false;
 +          this.notifyAll();
 +        }
 +      }
 +    }
 +
 +  }
 +
 +  boolean initiateMinorCompaction(MinorCompactionReason mincReason) {
 +    if (isClosed()) {
 +      // don't bother trying to get flush id if closed... could be closed after this check but that is ok... just trying to cut down on uneeded log messages....
 +      return false;
 +    }
 +
 +    // get the flush id before the new memmap is made available for write
 +    long flushId;
 +    try {
 +      flushId = getFlushID();
 +    } catch (NoNodeException e) {
 +      log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
 +      return false;
 +    }
 +    return initiateMinorCompaction(flushId, mincReason);
 +  }
 +
 +  boolean minorCompactNow(MinorCompactionReason mincReason) {
 +    long flushId;
 +    try {
 +      flushId = getFlushID();
 +    } catch (NoNodeException e) {
 +      log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
 +      return false;
 +    }
 +    MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
 +    if (mct == null)
 +      return false;
 +    mct.run();
 +    return true;
 +  }
 +
 +  boolean initiateMinorCompaction(long flushId, MinorCompactionReason mincReason) {
 +    MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
 +    if (mct == null)
 +      return false;
 +    tabletResources.executeMinorCompaction(mct);
 +    return true;
 +  }
 +
 +  private MinorCompactionTask createMinorCompactionTask(long flushId, MinorCompactionReason mincReason) {
 +    MinorCompactionTask mct;
 +    long t1, t2;
 +
 +    StringBuilder logMessage = null;
 +
 +    try {
 +      synchronized (this) {
 +        t1 = System.currentTimeMillis();
 +
 +        if (closing || closed || majorCompactionWaitingToStart || tabletMemory.memoryReservedForMinC() || tabletMemory.getMemTable().getNumEntries() == 0
 +            || updatingFlushID) {
 +
 +          logMessage = new StringBuilder();
 +
 +          logMessage.append(extent.toString());
 +          logMessage.append(" closing " + closing);
 +          logMessage.append(" closed " + closed);
 +          logMessage.append(" majorCompactionWaitingToStart " + majorCompactionWaitingToStart);
 +          if (tabletMemory != null)
 +            logMessage.append(" tabletMemory.memoryReservedForMinC() " + tabletMemory.memoryReservedForMinC());
 +          if (tabletMemory != null && tabletMemory.getMemTable() != null)
 +            logMessage.append(" tabletMemory.getMemTable().getNumEntries() " + tabletMemory.getMemTable().getNumEntries());
 +          logMessage.append(" updatingFlushID " + updatingFlushID);
 +
 +          return null;
 +        }
 +
 +        mct = prepareForMinC(flushId, mincReason);
 +        t2 = System.currentTimeMillis();
 +      }
 +    } finally {
 +      // log outside of sync block
 +      if (logMessage != null && log.isDebugEnabled())
 +        log.debug(logMessage);
 +    }
 +
 +    log.debug(String.format("MinC initiate lock %.2f secs", (t2 - t1) / 1000.0));
 +    return mct;
 +  }
 +
 +  long getFlushID() throws NoNodeException {
 +    try {
 +      String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +          + Constants.ZTABLE_FLUSH_ID;
-       return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++      return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
 +    } catch (InterruptedException e) {
 +      throw new RuntimeException(e);
 +    } catch (NumberFormatException nfe) {
 +      throw new RuntimeException(nfe);
 +    } catch (KeeperException ke) {
 +      if (ke instanceof NoNodeException) {
 +        throw (NoNodeException) ke;
 +      } else {
 +        throw new RuntimeException(ke);
 +      }
 +    }
 +  }
 +
 +  long getCompactionCancelID() {
 +    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
 +        + Constants.ZTABLE_COMPACT_CANCEL_ID;
 +
 +    try {
-       return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++      return Long.parseLong(new String(ZooReaderWriter.getInstance().get

<TRUNCATED>

[12/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 03635cc,0000000..9cd02eb
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -1,1051 -1,0 +1,1053 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.TreeMap;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.BatchWriterImpl;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Writer;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.Credentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.util.FastFormat;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +/**
 + * provides a reference to the metadata table for updates by tablet servers
 + */
 +public class MetadataTableUtil {
 +
 +  private static final Text EMPTY_TEXT = new Text();
 +  private static final byte[] EMPTY_BYTES = new byte[0];
 +  private static Map<Credentials,Writer> root_tables = new HashMap<Credentials,Writer>();
 +  private static Map<Credentials,Writer> metadata_tables = new HashMap<Credentials,Writer>();
 +  private static final Logger log = Logger.getLogger(MetadataTableUtil.class);
 +
 +  private MetadataTableUtil() {}
 +
 +  public synchronized static Writer getMetadataTable(Credentials credentials) {
 +    Writer metadataTable = metadata_tables.get(credentials);
 +    if (metadataTable == null) {
 +      metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID);
 +      metadata_tables.put(credentials, metadataTable);
 +    }
 +    return metadataTable;
 +  }
 +
 +  private synchronized static Writer getRootTable(Credentials credentials) {
 +    Writer rootTable = root_tables.get(credentials);
 +    if (rootTable == null) {
 +      rootTable = new Writer(HdfsZooInstance.getInstance(), credentials, RootTable.ID);
 +      root_tables.put(credentials, rootTable);
 +    }
 +    return rootTable;
 +  }
 +
 +  private static void putLockID(ZooLock zooLock, Mutation m) {
 +    TabletsSection.ServerColumnFamily.LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/")
-         .getBytes(Constants.UTF8)));
++        .getBytes(UTF_8)));
 +  }
 +
 +  private static void update(Credentials credentials, Mutation m, KeyExtent extent) {
 +    update(credentials, null, m, extent);
 +  }
 +
 +  public static void update(Credentials credentials, ZooLock zooLock, Mutation m, KeyExtent extent) {
 +    Writer t = extent.isMeta() ? getRootTable(credentials) : getMetadataTable(credentials);
 +    update(t, credentials, zooLock, m);
 +  }
 +
 +  public static void update(Writer t, Credentials credentials, ZooLock zooLock, Mutation m) {
 +    if (zooLock != null)
 +      putLockID(zooLock, m);
 +    while (true) {
 +      try {
 +        t.update(m);
 +        return;
 +      } catch (AccumuloException e) {
 +        log.error(e, e);
 +      } catch (AccumuloSecurityException e) {
 +        log.error(e, e);
 +      } catch (ConstraintViolationException e) {
 +        log.error(e, e);
 +        // retrying when a CVE occurs is probably futile and can cause problems, see ACCUMULO-3096
 +        throw new RuntimeException(e);
 +      } catch (TableNotFoundException e) {
 +        log.error(e, e);
 +      }
 +      UtilWaitThread.sleep(1000);
 +    }
 +  }
 +
 +  public static void updateTabletFlushID(KeyExtent extent, long flushID, Credentials credentials, ZooLock zooLock) {
 +    if (!extent.isRootTablet()) {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
-       TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(Constants.UTF8)));
++      TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(UTF_8)));
 +      update(credentials, zooLock, m, extent);
 +    }
 +  }
 +
 +  public static void updateTabletCompactID(KeyExtent extent, long compactID, Credentials credentials, ZooLock zooLock) {
 +    if (!extent.isRootTablet()) {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
-       TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(Constants.UTF8)));
++      TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(UTF_8)));
 +      update(credentials, zooLock, m, extent);
 +    }
 +  }
 +
 +  public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, Credentials credentials, ZooLock zooLock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
-     byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
++    byte[] tidBytes = Long.toString(tid).getBytes(UTF_8);
 +
 +    for (Entry<FileRef,DataFileValue> entry : estSizes.entrySet()) {
 +      Text file = entry.getKey().meta();
 +      m.put(DataFileColumnFamily.NAME, file, new Value(entry.getValue().encode()));
 +      m.put(TabletsSection.BulkFileColumnFamily.NAME, file, new Value(tidBytes));
 +    }
-     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
++    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
 +    update(credentials, zooLock, m, extent);
 +  }
 +
 +  public static void updateTabletDir(KeyExtent extent, String newDir, Credentials creds, ZooLock lock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
-     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
 +    update(creds, lock, m, extent);
 +  }
 +
 +  public static void addTablet(KeyExtent extent, String path, Credentials credentials, char timeType, ZooLock lock) {
 +    Mutation m = extent.getPrevRowUpdateMutation();
 +
-     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
-     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
++    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
++    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(UTF_8)));
 +
 +    update(credentials, lock, m, extent);
 +  }
 +
 +  public static void updateTabletPrevEndRow(KeyExtent extent, Credentials credentials) {
 +    Mutation m = extent.getPrevRowUpdateMutation(); //
 +    update(credentials, m, extent);
 +  }
 +
 +  public static void updateTabletVolumes(KeyExtent extent, List<LogEntry> logsToRemove, List<LogEntry> logsToAdd, List<FileRef> filesToRemove,
 +      SortedMap<FileRef,DataFileValue> filesToAdd, String newDir, ZooLock zooLock, Credentials credentials) {
 +
 +    if (extent.isRootTablet()) {
 +      if (newDir != null)
 +        throw new IllegalArgumentException("newDir not expected for " + extent);
 +
 +      if (filesToRemove.size() != 0 || filesToAdd.size() != 0)
 +        throw new IllegalArgumentException("files not expected for " + extent);
 +
 +      // add before removing in case of process death
 +      for (LogEntry logEntry : logsToAdd)
 +        addLogEntry(credentials, logEntry, zooLock);
 +
 +      removeUnusedWALEntries(extent, logsToRemove, zooLock);
 +    } else {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
 +
 +      for (LogEntry logEntry : logsToRemove)
 +        m.putDelete(logEntry.getColumnFamily(), logEntry.getColumnQualifier());
 +
 +      for (LogEntry logEntry : logsToAdd)
 +        m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
 +
 +      for (FileRef fileRef : filesToRemove)
 +        m.putDelete(DataFileColumnFamily.NAME, fileRef.meta());
 +
 +      for (Entry<FileRef,DataFileValue> entry : filesToAdd.entrySet())
 +        m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
 +
 +      if (newDir != null)
-         ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++        ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
 +
 +      update(credentials, m, extent);
 +    }
 +  }
 +
 +  public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, Credentials credentials) throws IOException {
 +    TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
 +
 +    Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
 +    mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    Text row = extent.getMetadataEntry();
 +    VolumeManager fs = VolumeManagerImpl.get();
 +
 +    Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text(""));
 +    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
 +
 +    mdScanner.setRange(new Range(new Key(row), endKey));
 +    for (Entry<Key,Value> entry : mdScanner) {
 +
 +      if (!entry.getKey().getRow().equals(row))
 +        break;
 +      DataFileValue dfv = new DataFileValue(entry.getValue().get());
 +      sizes.put(new FileRef(fs, entry.getKey()), dfv);
 +    }
 +
 +    return sizes;
 +  }
 +
 +  public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, Credentials credentials, ZooLock zooLock) {
 +    KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
 +    Mutation m = ke.getPrevRowUpdateMutation();
 +    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
 +    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
 +    update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
 +  }
 +
 +  public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, Credentials credentials, ZooLock zooLock) {
 +    Mutation m = extent.getPrevRowUpdateMutation(); //
 +
-     TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
++    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(UTF_8)));
 +
 +    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
 +    ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
 +    update(credentials, zooLock, m, extent);
 +  }
 +
 +  public static void finishSplit(Text metadataEntry, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
 +      ZooLock zooLock) {
 +    Mutation m = new Mutation(metadataEntry);
 +    TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
 +    TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
 +    ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
 +
 +    for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
 +      m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
 +    }
 +
 +    for (FileRef pathToRemove : highDatafilesToRemove) {
 +      m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta());
 +    }
 +
 +    update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
 +  }
 +
 +  public static void finishSplit(KeyExtent extent, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
 +      ZooLock zooLock) {
 +    finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
 +  }
 +
 +  public static void addDeleteEntries(KeyExtent extent, Set<FileRef> datafilesToDelete, Credentials credentials) throws IOException {
 +
 +    String tableId = extent.getTableId().toString();
 +
 +    // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
 +    for (FileRef pathToRemove : datafilesToDelete) {
 +      update(credentials, createDeleteMutation(tableId, pathToRemove.path().toString()), extent);
 +    }
 +  }
 +
 +  public static void addDeleteEntry(String tableId, String path) throws IOException {
 +    update(SystemCredentials.get(), createDeleteMutation(tableId, path), new KeyExtent(new Text(tableId), null, null));
 +  }
 +
 +  public static Mutation createDeleteMutation(String tableId, String pathToRemove) throws IOException {
 +    Path path = VolumeManagerImpl.get().getFullPath(tableId, pathToRemove);
 +    Mutation delFlag = new Mutation(new Text(MetadataSchema.DeletesSection.getRowPrefix() + path.toString()));
 +    delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
 +    return delFlag;
 +  }
 +
 +  public static void removeScanFiles(KeyExtent extent, Set<FileRef> scanFiles, Credentials credentials, ZooLock zooLock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
 +
 +    for (FileRef pathToRemove : scanFiles)
 +      m.putDelete(ScanFileColumnFamily.NAME, pathToRemove.meta());
 +
 +    update(credentials, zooLock, m, extent);
 +  }
 +
 +  public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<FileRef,FileUtil.FileInfo> firstAndLastRows,
 +      SortedMap<FileRef,DataFileValue> datafiles, SortedMap<FileRef,DataFileValue> lowDatafileSizes, SortedMap<FileRef,DataFileValue> highDatafileSizes,
 +      List<FileRef> highDatafilesToRemove) {
 +
 +    for (Entry<FileRef,DataFileValue> entry : datafiles.entrySet()) {
 +
 +      Text firstRow = null;
 +      Text lastRow = null;
 +
 +      boolean rowsKnown = false;
 +
 +      FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
 +
 +      if (mfi != null) {
 +        firstRow = mfi.getFirstRow();
 +        lastRow = mfi.getLastRow();
 +        rowsKnown = true;
 +      }
 +
 +      if (rowsKnown && firstRow.compareTo(midRow) > 0) {
 +        // only in high
 +        long highSize = entry.getValue().getSize();
 +        long highEntries = entry.getValue().getNumEntries();
 +        highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
 +      } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
 +        // only in low
 +        long lowSize = entry.getValue().getSize();
 +        long lowEntries = entry.getValue().getNumEntries();
 +        lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
 +
 +        highDatafilesToRemove.add(entry.getKey());
 +      } else {
 +        long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
 +        long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
 +        lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
 +
 +        long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
 +        long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
 +        highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
 +      }
 +    }
 +  }
 +
 +  public static void deleteTable(String tableId, boolean insertDeletes, Credentials credentials, ZooLock lock) throws AccumuloException, IOException {
 +    Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
 +    Text tableIdText = new Text(tableId);
 +    BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000)
 +        .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
 +
 +    // scan metadata for our table and delete everything we find
 +    Mutation m = null;
 +    ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
 +
 +    // insert deletes before deleting data from metadata... this makes the code fault tolerant
 +    if (insertDeletes) {
 +
 +      ms.fetchColumnFamily(DataFileColumnFamily.NAME);
 +      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms);
 +
 +      for (Entry<Key,Value> cell : ms) {
 +        Key key = cell.getKey();
 +
 +        if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +          FileRef ref = new FileRef(VolumeManagerImpl.get(), key);
 +          bw.addMutation(createDeleteMutation(tableId, ref.meta().toString()));
 +        }
 +
 +        if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
 +          bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
 +        }
 +      }
 +
 +      bw.flush();
 +
 +      ms.clearColumns();
 +    }
 +
 +    for (Entry<Key,Value> cell : ms) {
 +      Key key = cell.getKey();
 +
 +      if (m == null) {
 +        m = new Mutation(key.getRow());
 +        if (lock != null)
 +          putLockID(lock, m);
 +      }
 +
 +      if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
 +        bw.addMutation(m);
 +        m = new Mutation(key.getRow());
 +        if (lock != null)
 +          putLockID(lock, m);
 +      }
 +      m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
 +    }
 +
 +    if (m != null)
 +      bw.addMutation(m);
 +
 +    bw.close();
 +  }
 +
 +  static String getZookeeperLogLocation() {
 +    return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_WALOGS;
 +  }
 +
 +  public static void addLogEntry(Credentials credentials, LogEntry entry, ZooLock zooLock) {
 +    if (entry.extent.isRootTablet()) {
 +      String root = getZookeeperLogLocation();
 +      while (true) {
 +        try {
 +          IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +          if (zoo.isLockHeld(zooLock.getLockID())) {
 +            String[] parts = entry.filename.split("/");
 +            String uniqueId = parts[parts.length - 1];
 +            zoo.putPersistentData(root + "/" + uniqueId, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
 +          }
 +          break;
 +        } catch (KeeperException e) {
 +          log.error(e, e);
 +        } catch (InterruptedException e) {
 +          log.error(e, e);
 +        } catch (IOException e) {
 +          log.error(e, e);
 +        }
 +        UtilWaitThread.sleep(1000);
 +      }
 +    } else {
 +      Mutation m = new Mutation(entry.getRow());
 +      m.put(entry.getColumnFamily(), entry.getColumnQualifier(), entry.getValue());
 +      update(credentials, zooLock, m, entry.extent);
 +    }
 +  }
 +
 +  public static void setRootTabletDir(String dir) throws IOException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
 +    try {
-       zoo.putPersistentData(zpath, dir.getBytes(Constants.UTF8), -1, NodeExistsPolicy.OVERWRITE);
++      zoo.putPersistentData(zpath, dir.getBytes(UTF_8), -1, NodeExistsPolicy.OVERWRITE);
 +    } catch (KeeperException e) {
 +      throw new IOException(e);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  public static String getRootTabletDir() throws IOException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
 +    try {
-       return new String(zoo.getData(zpath, null), Constants.UTF8);
++      return new String(zoo.getData(zpath, null), UTF_8);
 +    } catch (KeeperException e) {
 +      throw new IOException(e);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  public static Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> getFileAndLogEntries(Credentials credentials, KeyExtent extent) throws KeeperException,
 +      InterruptedException, IOException {
 +    ArrayList<LogEntry> result = new ArrayList<LogEntry>();
 +    TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
 +
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    if (extent.isRootTablet()) {
 +      getRootLogEntries(result);
 +      Path rootDir = new Path(getRootTabletDir());
 +      FileStatus[] files = fs.listStatus(rootDir);
 +      for (FileStatus fileStatus : files) {
 +        if (fileStatus.getPath().toString().endsWith("_tmp")) {
 +          continue;
 +        }
 +        DataFileValue dfv = new DataFileValue(0, 0);
 +        sizes.put(new FileRef(fileStatus.getPath().toString(), fileStatus.getPath()), dfv);
 +      }
 +
 +    } else {
 +      String systemTableToCheck = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
 +      Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, systemTableToCheck, Authorizations.EMPTY);
 +      scanner.fetchColumnFamily(LogColumnFamily.NAME);
 +      scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +      scanner.setRange(extent.toMetadataRange());
 +
 +      for (Entry<Key,Value> entry : scanner) {
 +        if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
 +          throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
 +        }
 +
 +        if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
 +          result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
 +        } else if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +          DataFileValue dfv = new DataFileValue(entry.getValue().get());
 +          sizes.put(new FileRef(fs, entry.getKey()), dfv);
 +        } else {
 +          throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
 +        }
 +      }
 +    }
 +
 +    return new Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>>(result, sizes);
 +  }
 +
 +  public static List<LogEntry> getLogEntries(Credentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
 +    log.info("Scanning logging entries for " + extent);
 +    ArrayList<LogEntry> result = new ArrayList<LogEntry>();
 +    if (extent.equals(RootTable.EXTENT)) {
 +      log.info("Getting logs for root tablet from zookeeper");
 +      getRootLogEntries(result);
 +    } else {
 +      log.info("Scanning metadata for logs used for tablet " + extent);
 +      Scanner scanner = getTabletLogScanner(credentials, extent);
 +      Text pattern = extent.getMetadataEntry();
 +      for (Entry<Key,Value> entry : scanner) {
 +        Text row = entry.getKey().getRow();
 +        if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
 +          if (row.equals(pattern)) {
 +            result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
 +          }
 +        }
 +      }
 +    }
 +
 +    Collections.sort(result, new Comparator<LogEntry>() {
 +      @Override
 +      public int compare(LogEntry o1, LogEntry o2) {
 +        long diff = o1.timestamp - o2.timestamp;
 +        if (diff < 0)
 +          return -1;
 +        if (diff > 0)
 +          return 1;
 +        return 0;
 +      }
 +    });
 +    log.info("Returning logs " + result + " for extent " + extent);
 +    return result;
 +  }
 +
 +  static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    String root = getZookeeperLogLocation();
 +    // there's a little race between getting the children and fetching
 +    // the data. The log can be removed in between.
 +    while (true) {
 +      result.clear();
 +      for (String child : zoo.getChildren(root)) {
 +        LogEntry e = new LogEntry();
 +        try {
 +          e.fromBytes(zoo.getData(root + "/" + child, null));
 +          // upgrade from !0;!0<< -> +r<<
 +          e.extent = RootTable.EXTENT;
 +          result.add(e);
 +        } catch (KeeperException.NoNodeException ex) {
 +          continue;
 +        }
 +      }
 +      break;
 +    }
 +  }
 +
 +  private static Scanner getTabletLogScanner(Credentials credentials, KeyExtent extent) {
 +    String tableId = MetadataTable.ID;
 +    if (extent.isMeta())
 +      tableId = RootTable.ID;
 +    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, tableId, Authorizations.EMPTY);
 +    scanner.fetchColumnFamily(LogColumnFamily.NAME);
 +    Text start = extent.getMetadataEntry();
 +    Key endKey = new Key(start, LogColumnFamily.NAME);
 +    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
 +    scanner.setRange(new Range(new Key(start), endKey));
 +    return scanner;
 +  }
 +
 +  private static class LogEntryIterator implements Iterator<LogEntry> {
 +
 +    Iterator<LogEntry> zookeeperEntries = null;
 +    Iterator<LogEntry> rootTableEntries = null;
 +    Iterator<Entry<Key,Value>> metadataEntries = null;
 +
 +    LogEntryIterator(Credentials creds) throws IOException, KeeperException, InterruptedException {
 +      zookeeperEntries = getLogEntries(creds, RootTable.EXTENT).iterator();
 +      rootTableEntries = getLogEntries(creds, new KeyExtent(new Text(MetadataTable.ID), null, null)).iterator();
 +      try {
 +        Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken())
 +            .createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +        log.info("Setting range to " + MetadataSchema.TabletsSection.getRange());
 +        scanner.setRange(MetadataSchema.TabletsSection.getRange());
 +        scanner.fetchColumnFamily(LogColumnFamily.NAME);
 +        metadataEntries = scanner.iterator();
 +      } catch (Exception ex) {
 +        throw new IOException(ex);
 +      }
 +    }
 +
 +    @Override
 +    public boolean hasNext() {
 +      return zookeeperEntries.hasNext() || rootTableEntries.hasNext() || metadataEntries.hasNext();
 +    }
 +
 +    @Override
 +    public LogEntry next() {
 +      if (zookeeperEntries.hasNext()) {
 +        return zookeeperEntries.next();
 +      }
 +      if (rootTableEntries.hasNext()) {
 +        return rootTableEntries.next();
 +      }
 +      Entry<Key,Value> entry = metadataEntries.next();
 +      return LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
 +    }
 +
 +    @Override
 +    public void remove() {
 +      throw new UnsupportedOperationException();
 +    }
 +  }
 +
 +  public static Iterator<LogEntry> getLogEntries(Credentials creds) throws IOException, KeeperException, InterruptedException {
 +    return new LogEntryIterator(creds);
 +  }
 +
 +  public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
 +    if (extent.isRootTablet()) {
 +      for (LogEntry entry : logEntries) {
 +        String root = getZookeeperLogLocation();
 +        while (true) {
 +          try {
 +            IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +            if (zoo.isLockHeld(zooLock.getLockID())) {
 +              String parts[] = entry.filename.split("/");
 +              zoo.recursiveDelete(root + "/" + parts[parts.length - 1], NodeMissingPolicy.SKIP);
 +            }
 +            break;
 +          } catch (Exception e) {
 +            log.error(e, e);
 +          }
 +          UtilWaitThread.sleep(1000);
 +        }
 +      }
 +    } else {
 +      Mutation m = new Mutation(extent.getMetadataEntry());
 +      for (LogEntry entry : logEntries) {
 +        m.putDelete(LogColumnFamily.NAME, new Text(entry.getName()));
 +      }
 +      update(SystemCredentials.get(), zooLock, m, extent);
 +    }
 +  }
 +
 +  private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
 +    for (Entry<Key,Value> entry : tablet.entrySet()) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        String cf = entry.getKey().getColumnQualifier().toString();
 +        if (srcTableId != null && !cf.startsWith("../") && !cf.contains(":")) {
 +          cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
 +        }
 +        files.add(cf);
 +      }
 +    }
 +  }
 +
 +  private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
 +
 +    KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
 +    Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
 +
 +    for (Entry<Key,Value> entry : tablet.entrySet()) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        String cf = entry.getKey().getColumnQualifier().toString();
 +        if (!cf.startsWith("../") && !cf.contains(":"))
 +          cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
 +        m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
 +      } else if (entry.getKey().getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
 +        m.put(TabletsSection.LastLocationColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
 +      } else if (entry.getKey().getColumnFamily().equals(TabletsSection.LastLocationColumnFamily.NAME)) {
 +        // skip
 +      } else {
 +        m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
 +      }
 +    }
 +    return m;
 +  }
 +
 +  private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
 +    String tableName = MetadataTable.NAME;
 +    if (tableId.equals(MetadataTable.ID))
 +      tableName = RootTable.NAME;
 +    Scanner mscanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(DataFileColumnFamily.NAME);
 +    mscanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
 +    mscanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
 +    mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
 +    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(mscanner);
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(mscanner);
 +    return mscanner;
 +  }
 +
 +  static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
 +    TabletIterator ti;
 +    if (srcTableId.equals(MetadataTable.ID))
 +      ti = new TabletIterator(createCloneScanner(srcTableId, conn), new Range(), true, true);
 +    else
 +      ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
 +
 +    if (!ti.hasNext())
 +      throw new RuntimeException(" table deleted during clone?  srcTableId = " + srcTableId);
 +
 +    while (ti.hasNext())
 +      bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
 +
 +    bw.flush();
 +  }
 +
 +  private static int compareEndRows(Text endRow1, Text endRow2) {
 +    return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
 +  }
 +
 +  static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
 +    TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
 +        true);
 +    TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
 +
 +    if (!cloneIter.hasNext() || !srcIter.hasNext())
 +      throw new RuntimeException(" table deleted during clone?  srcTableId = " + srcTableId + " tableId=" + tableId);
 +
 +    int rewrites = 0;
 +
 +    while (cloneIter.hasNext()) {
 +      Map<Key,Value> cloneTablet = cloneIter.next();
 +      Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
 +      HashSet<String> cloneFiles = new HashSet<String>();
 +
 +      boolean cloneSuccessful = false;
 +      for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
 +        if (entry.getKey().getColumnFamily().equals(ClonedColumnFamily.NAME)) {
 +          cloneSuccessful = true;
 +          break;
 +        }
 +      }
 +
 +      if (!cloneSuccessful)
 +        getFiles(cloneFiles, cloneTablet, null);
 +
 +      List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
 +      Map<Key,Value> srcTablet = srcIter.next();
 +      srcTablets.add(srcTablet);
 +
 +      Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
 +
 +      int cmp = compareEndRows(cloneEndRow, srcEndRow);
 +      if (cmp < 0)
 +        throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
 +
 +      HashSet<String> srcFiles = new HashSet<String>();
 +      if (!cloneSuccessful)
 +        getFiles(srcFiles, srcTablet, srcTableId);
 +
 +      while (cmp > 0) {
 +        srcTablet = srcIter.next();
 +        srcTablets.add(srcTablet);
 +        srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
 +        cmp = compareEndRows(cloneEndRow, srcEndRow);
 +        if (cmp < 0)
 +          throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
 +
 +        if (!cloneSuccessful)
 +          getFiles(srcFiles, srcTablet, srcTableId);
 +      }
 +
 +      if (cloneSuccessful)
 +        continue;
 +
 +      if (!srcFiles.containsAll(cloneFiles)) {
 +        // delete existing cloned tablet entry
 +        Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
 +
 +        for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
 +          Key k = entry.getKey();
 +          m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
 +        }
 +
 +        bw.addMutation(m);
 +
 +        for (Map<Key,Value> st : srcTablets)
 +          bw.addMutation(createCloneMutation(srcTableId, tableId, st));
 +
 +        rewrites++;
 +      } else {
 +        // write out marker that this tablet was successfully cloned
 +        Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
-         m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
++        m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(UTF_8)));
 +        bw.addMutation(m);
 +      }
 +    }
 +
 +    bw.flush();
 +    return rewrites;
 +  }
 +
 +  public static void cloneTable(Instance instance, String srcTableId, String tableId, VolumeManager volumeManager) throws Exception {
 +
 +    Connector conn = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
 +    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +
 +    while (true) {
 +
 +      try {
 +        initializeClone(srcTableId, tableId, conn, bw);
 +
 +        // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
 +
 +        while (true) {
 +          int rewrites = checkClone(srcTableId, tableId, conn, bw);
 +
 +          if (rewrites == 0)
 +            break;
 +        }
 +
 +        bw.flush();
 +        break;
 +
 +      } catch (TabletIterator.TabletDeletedException tde) {
 +        // tablets were merged in the src table
 +        bw.flush();
 +
 +        // delete what we have cloned and try again
 +        deleteTable(tableId, false, SystemCredentials.get(), null);
 +
 +        log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
 +
 +        UtilWaitThread.sleep(100);
 +      }
 +    }
 +
 +    // delete the clone markers and create directory entries
 +    Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
 +
 +    int dirCount = 0;
 +
 +    for (Entry<Key,Value> entry : mscanner) {
 +      Key k = entry.getKey();
 +      Mutation m = new Mutation(k.getRow());
 +      m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
 +      String dir = volumeManager.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId
 +          + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
-       TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(Constants.UTF8)));
++      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8)));
 +
 +      bw.addMutation(m);
 +    }
 +
 +    bw.close();
 +
 +  }
 +
 +  public static void chopped(KeyExtent extent, ZooLock zooLock) {
 +    Mutation m = new Mutation(extent.getMetadataEntry());
-     ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
++    ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(UTF_8)));
 +    update(SystemCredentials.get(), zooLock, m, extent);
 +  }
 +
 +  public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
 +    Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 +    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
 +    mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 +    for (Entry<Key,Value> entry : mscanner) {
 +      log.debug("Looking at entry " + entry + " with tid " + tid);
 +      if (Long.parseLong(entry.getValue().toString()) == tid) {
 +        log.debug("deleting entry " + entry);
 +        Mutation m = new Mutation(entry.getKey().getRow());
 +        m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
 +        bw.addMutation(m);
 +      }
 +    }
 +    bw.close();
 +  }
 +
 +  public static List<FileRef> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) throws IOException {
 +    List<FileRef> result = new ArrayList<FileRef>();
 +    try {
 +      VolumeManager fs = VolumeManagerImpl.get();
 +      Scanner mscanner = new IsolatedScanner(conn.createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY));
 +      mscanner.setRange(extent.toMetadataRange());
 +      mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +      for (Entry<Key,Value> entry : mscanner) {
 +        if (Long.parseLong(entry.getValue().toString()) == tid) {
 +          result.add(new FileRef(fs, entry.getKey()));
 +        }
 +      }
 +      return result;
 +    } catch (TableNotFoundException ex) {
 +      // unlikely
 +      throw new RuntimeException("Onos! teh metadata table has vanished!!");
 +    }
 +  }
 +
 +  public static Map<FileRef,Long> getBulkFilesLoaded(Credentials credentials, KeyExtent extent) throws IOException {
 +    Text metadataRow = extent.getMetadataEntry();
 +    Map<FileRef,Long> ret = new HashMap<FileRef,Long>();
 +
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, extent.isMeta() ? RootTable.ID : MetadataTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(new Range(metadataRow));
 +    scanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
 +    for (Entry<Key,Value> entry : scanner) {
 +      Long tid = Long.parseLong(entry.getValue().toString());
 +      ret.put(new FileRef(fs, entry.getKey()), tid);
 +    }
 +    return ret;
 +  }
 +
 +  public static void addBulkLoadInProgressFlag(String path) {
 +
 +    Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
 +    m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
 +
 +    // new KeyExtent is only added to force update to write to the metadata table, not the root table
 +    // because bulk loads aren't supported to the metadata table
 +    update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
 +  }
 +
 +  public static void removeBulkLoadInProgressFlag(String path) {
 +
 +    Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
 +    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
 +
 +    // new KeyExtent is only added to force update to write to the metadata table, not the root table
 +    // because bulk loads aren't supported to the metadata table
 +    update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
 +  }
 +
 +  /**
 +   * During an upgrade we need to move deletion requests for files under the !METADATA table to the root tablet.
 +   */
 +  public static void moveMetaDeleteMarkers(Instance instance, Credentials creds) {
 +    String oldDeletesPrefix = "!!~del";
 +    Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false);
 +
 +    // move old delete markers to new location, to standardize table schema between all metadata tables
 +    Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(oldDeletesRange);
 +    for (Entry<Key,Value> entry : scanner) {
 +      String row = entry.getKey().getRow().toString();
 +      if (row.startsWith(oldDeletesPrefix)) {
 +        moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row, oldDeletesPrefix);
 +      } else {
 +        break;
 +      }
 +    }
 +
 +  }
 +
 +  public static void moveMetaDeleteMarkersFrom14(Instance instance, Credentials creds) {
 +    // new KeyExtent is only added to force update to write to the metadata table, not the root table
 +    KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"), null, null);
 +
 +    // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
 +    Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID, Authorizations.EMPTY);
 +    scanner.setRange(MetadataSchema.DeletesSection.getRange());
 +    for (Entry<Key,Value> entry : scanner) {
 +      String row = entry.getKey().getRow().toString();
 +      if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" + MetadataTable.ID)) {
 +        moveDeleteEntry(creds, notMetadata, entry, row, MetadataSchema.DeletesSection.getRowPrefix());
 +      } else {
 +        break;
 +      }
 +    }
 +  }
 +
 +  private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent, Entry<Key,Value> entry, String rowID, String prefix) {
 +    String filename = rowID.substring(prefix.length());
 +
 +    // add the new entry first
 +    log.info("Moving " + filename + " marker in " + RootTable.NAME);
 +    Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
 +    m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
 +    update(creds, m, RootTable.EXTENT);
 +
 +    // then remove the old entry
 +    m = new Mutation(entry.getKey().getRow());
 +    m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
 +    update(creds, m, oldExtent);
 +  }
 +
 +  public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
 +    TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
 +
 +    HashSet<ColumnFQ> colSet = null;
 +    if (columns != null) {
 +      colSet = new HashSet<ColumnFQ>(columns);
 +    }
 +
 +    for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
 +
 +      if (columns != null && !colSet.contains(new ColumnFQ(entry.getKey()))) {
 +        continue;
 +      }
 +
 +      Text row = entry.getKey().getRow();
 +
 +      SortedMap<ColumnFQ,Value> colVals = tabletEntries.get(row);
 +      if (colVals == null) {
 +        colVals = new TreeMap<ColumnFQ,Value>();
 +        tabletEntries.put(row, colVals);
 +      }
 +
 +      colVals.put(new ColumnFQ(entry.getKey()), entry.getValue());
 +    }
 +
 +    return tabletEntries;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index 4e5df9e,0000000..c0d6894
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class NamespacePropUtil {
 +  public static boolean setNamespaceProperty(String namespaceId, String property, String value) throws KeeperException, InterruptedException {
 +    if (!isPropertyValid(property, value))
 +      return false;
 +
 +    // create the zk node for per-namespace properties for this namespace if it doesn't already exist
 +    String zkNamespacePath = getPath(namespaceId);
 +    ZooReaderWriter.getInstance().putPersistentData(zkNamespacePath, new byte[0], NodeExistsPolicy.SKIP);
 +
 +    // create the zk node for this property and set it's data to the specified value
 +    String zPath = zkNamespacePath + "/" + property;
-     ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +
 +    return true;
 +  }
 +
 +  public static boolean isPropertyValid(String property, String value) {
 +    Property p = Property.getPropertyByKey(property);
 +    if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
 +      return false;
 +
 +    return true;
 +  }
 +
 +  public static void removeNamespaceProperty(String namespaceId, String property) throws InterruptedException, KeeperException {
 +    String zPath = getPath(namespaceId) + "/" + property;
 +    ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
 +  }
 +
 +  private static String getPath(String namespaceId) {
 +    return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZNAMESPACES + "/" + namespaceId + Constants.ZNAMESPACE_CONF;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index dd540f2,0000000..8924c85
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -1,131 -1,0 +1,131 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.util.Map.Entry;
 +
- import org.apache.accumulo.core.Constants;
- 
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.tables.TableManager;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.log4j.Logger;
 +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 +
 +public class RandomizeVolumes {
 +  private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
 +
 +  public static void main(String[] args) throws AccumuloException, AccumuloSecurityException {
 +    ClientOnRequiredTable opts = new ClientOnRequiredTable();
 +    opts.parseArgs(RandomizeVolumes.class.getName(), args);
 +    Connector c;
 +    if (opts.getToken() == null) {
 +      SystemCredentials creds = SystemCredentials.get();
 +      c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
 +    } else {
 +      c = opts.getConnector();
 +    }
 +    try {
 +      int status = randomize(c, opts.tableName);
 +      System.exit(status);
 +    } catch (Exception ex) {
 +      log.error(ex, ex);
 +      System.exit(4);
 +    }
 +  }
 +
 +  public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
 +    VolumeManager vm = VolumeManagerImpl.get();
 +    if (vm.getVolumes().size() < 2) {
 +      log.error("There are not enough volumes configured");
 +      return 1;
 +    }
 +    String tableId = c.tableOperations().tableIdMap().get(tableName);
 +    if (null == tableId) {
 +      log.error("Could not determine the table ID for table " + tableName);
 +      return 2;
 +    }
 +    TableState tableState = TableManager.getInstance().getTableState(tableId);
 +    if (TableState.OFFLINE != tableState) {
 +      log.info("Taking " + tableName + " offline");
 +      c.tableOperations().offline(tableName, true);
 +      log.info(tableName + " offline");
 +    }
 +    log.info("Rewriting entries for " + tableName);
 +    Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +    DIRECTORY_COLUMN.fetch(scanner);
 +    scanner.setRange(TabletsSection.getRange(tableId));
 +    BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      String oldLocation = entry.getValue().toString();
 +      String directory;
 +      if (oldLocation.contains(":")) {
 +        String[] parts = oldLocation.split(Path.SEPARATOR);
 +        String tableIdEntry = parts[parts.length - 2];
 +        if (!tableIdEntry.equals(tableId)) {
 +          log.error("Unexpected table id found: " + tableIdEntry + ", expected " + tableId + "; skipping");
 +          continue;
 +        }
 +        directory = parts[parts.length - 1];
 +      } else {
 +        directory = oldLocation.substring(Path.SEPARATOR.length());
 +      }
 +      Key key = entry.getKey();
 +      Mutation m = new Mutation(key.getRow());
 +      
 +      String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
-       m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(Constants.UTF8)));
++      m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
 +      if (log.isTraceEnabled()) {
 +        log.trace("Replacing " + oldLocation + " with " + newLocation);
 +      }
 +      writer.addMutation(m);
 +      try {
 +        vm.mkdirs(new Path(newLocation));
 +      } catch (IOException ex) {
 +        // nevermind
 +      }
 +      count++;
 +    }
 +    writer.close();
 +    log.info("Updated " + count + " entries for table " + tableName);
 +    if (TableState.OFFLINE != tableState) {
 +      c.tableOperations().online(tableName, true);
 +      log.info("table " + tableName + " back online");
 +    }
 +    return 0;
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index a08000e,0000000..a82fa14
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,124 -1,0 +1,125 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.FileInputStream;
 +import java.io.InputStream;
 +import java.util.Stack;
 +
 +import javax.xml.parsers.SAXParser;
 +import javax.xml.parsers.SAXParserFactory;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +import org.xml.sax.Attributes;
 +import org.xml.sax.SAXException;
 +import org.xml.sax.helpers.DefaultHandler;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class RestoreZookeeper {
 +  
 +  private static class Restore extends DefaultHandler {
 +    IZooReaderWriter zk = null;
 +    Stack<String> cwd = new Stack<String>();
 +    boolean overwrite = false;
 +    
 +    Restore(IZooReaderWriter zk, boolean overwrite) {
 +      this.zk = zk;
 +      this.overwrite = overwrite;
 +    }
 +    
 +    @Override
 +    public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
 +      if ("node".equals(name)) {
 +        String child = attributes.getValue("name");
 +        if (child == null)
 +          throw new RuntimeException("name attribute not set");
 +        String encoding = attributes.getValue("encoding");
 +        String value = attributes.getValue("value");
 +        if (value == null)
 +          value = "";
 +        String path = cwd.lastElement() + "/" + child;
 +        create(path, value, encoding);
 +        cwd.push(path);
 +      } else if ("dump".equals(name)) {
 +        String root = attributes.getValue("root");
 +        if (root.equals("/"))
 +          cwd.push("");
 +        else
 +          cwd.push(root);
-         create(root, "", Constants.UTF8.name());
++        create(root, "", UTF_8.name());
 +      }
 +    }
 +    
 +    @Override
 +    public void endElement(String uri, String localName, String name) throws SAXException {
 +      cwd.pop();
 +    }
 +    
 +    // assume UTF-8 if not "base64"
 +    private void create(String path, String value, String encoding) {
-       byte[] data = value.getBytes(Constants.UTF8);
++      byte[] data = value.getBytes(UTF_8);
 +      if ("base64".equals(encoding))
 +        data = Base64.decodeBase64(data);
 +      try {
 +        try {
 +          zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
 +        } catch (KeeperException e) {
 +          if (e.code().equals(KeeperException.Code.NODEEXISTS))
 +            throw new RuntimeException(path + " exists.  Remove it first.");
 +          throw e;
 +        }
 +      } catch (Exception e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +  }
 +  
 +  static class Opts extends Help {
 +    @Parameter(names = {"-z", "--keepers"})
 +    String keepers = "localhost:2181";
 +    @Parameter(names = "--overwrite")
 +    boolean overwrite = false;
 +    @Parameter(names = "--file")
 +    String file;
 +  }
 +  
 +  public static void main(String[] args) throws Exception {
 +    Logger.getRootLogger().setLevel(Level.WARN);
 +    Opts opts = new Opts();
 +    opts.parseArgs(RestoreZookeeper.class.getName(), args);
 +    
 +    InputStream in = System.in;
 +    if (opts.file != null) {
 +      in = new FileInputStream(opts.file);
 +    }
 +    
 +    SAXParserFactory factory = SAXParserFactory.newInstance();
 +    SAXParser parser = factory.newSAXParser();
 +    parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
 +    in.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index 85f29ba,0000000..b1a553d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,277 -1,0 +1,278 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.File;
 +import java.io.FileInputStream;
 +import java.io.FileNotFoundException;
 +import java.io.FilenameFilter;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.io.UnsupportedEncodingException;
 +import java.net.Socket;
 +import java.net.URLEncoder;
 +import java.text.ParseException;
 +import java.text.SimpleDateFormat;
 +import java.util.Calendar;
 +import java.util.Date;
 +import java.util.Map;
 +import java.util.regex.Matcher;
 +import java.util.regex.Pattern;
 +
 +import javax.net.SocketFactory;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.commons.io.filefilter.WildcardFileFilter;
 +import org.apache.commons.lang.math.LongRange;
 +import org.apache.log4j.Category;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +import org.apache.log4j.spi.Filter;
 +import org.apache.log4j.spi.LocationInfo;
 +import org.apache.log4j.spi.LoggingEvent;
 +import org.apache.log4j.spi.ThrowableInformation;
 +import org.apache.log4j.varia.LevelRangeFilter;
 +import org.apache.log4j.xml.XMLLayout;
 +
 +import com.beust.jcommander.IStringConverter;
 +import com.beust.jcommander.Parameter;
 +
 +public class SendLogToChainsaw extends XMLLayout {
 +
 +  private static Pattern logPattern = Pattern.compile(
 +      "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
 +
 +  private File[] logFiles = null;
 +
 +  private SocketFactory factory = SocketFactory.getDefault();
 +
 +  private WildcardFileFilter fileFilter = null;
 +
 +  private Socket socket = null;
 +
 +  private Pattern lineFilter = null;
 +
 +  private LongRange dateFilter = null;
 +
 +  private LevelRangeFilter levelFilter = null;
 +
 +  public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
 +
 +    // Set up the file name filter
 +    if (null != fileNameFilter) {
 +      fileFilter = new WildcardFileFilter(fileNameFilter);
 +    } else {
 +      fileFilter = new WildcardFileFilter("*");
 +    }
 +
 +    // Get the list of files that match
 +    File dir = new File(directory);
 +    if (dir.isDirectory()) {
 +      logFiles = dir.listFiles((FilenameFilter) fileFilter);
 +    } else {
 +      throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
 +    }
 +
 +    if (logFiles.length == 0) {
 +      throw new IllegalArgumentException("No files match the supplied filter.");
 +    }
 +
 +    socket = factory.createSocket(host, port);
 +
 +    lineFilter = Pattern.compile(regex);
 +
 +    // Create Date Filter
 +    if (null != start) {
 +      if (end == null)
 +        end = new Date(System.currentTimeMillis());
 +      dateFilter = new LongRange(start.getTime(), end.getTime());
 +    }
 +
 +    if (null != level) {
 +      Level base = Level.toLevel(level.toUpperCase());
 +      levelFilter = new LevelRangeFilter();
 +      levelFilter.setAcceptOnMatch(true);
 +      levelFilter.setLevelMin(base);
 +      levelFilter.setLevelMax(Level.FATAL);
 +    }
 +  }
 +
 +  public void processLogFiles() throws IOException {
 +    String line = null;
 +    String out = null;
 +    InputStreamReader isReader = null;
 +    BufferedReader reader = null;
 +    try {
 +      for (File log : logFiles) {
 +        // Parse the server type and name from the log file name
 +        String threadName = log.getName().substring(0, log.getName().indexOf("."));
 +        try {
-           isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
++          isReader = new InputStreamReader(new FileInputStream(log), UTF_8);
 +        } catch (FileNotFoundException e) {
 +          System.out.println("Unable to find file: " + log.getAbsolutePath());
 +          throw e;
 +        }
 +        reader = new BufferedReader(isReader);
 +
 +        try {
 +          line = reader.readLine();
 +          while (null != line) {
 +            out = convertLine(line, threadName);
 +            if (null != out) {
 +              if (socket != null && socket.isConnected())
-                 socket.getOutputStream().write(out.getBytes(Constants.UTF8));
++                socket.getOutputStream().write(out.getBytes(UTF_8));
 +              else
 +                System.err.println("Unable to send data to transport");
 +            }
 +            line = reader.readLine();
 +          }
 +        } catch (IOException e) {
 +          System.out.println("Error processing line: " + line + ". Output was " + out);
 +          throw e;
 +        } finally {
 +          if (reader != null) {
 +            reader.close();
 +          }
 +          if (isReader != null) {
 +            isReader.close();
 +          }
 +        }
 +      }
 +    } finally {
 +      if (socket != null && socket.isConnected()) {
 +        socket.close();
 +      }
 +    }
 +  }
 +
 +  private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
 +    String result = null;
 +    Matcher m = logPattern.matcher(line);
 +    if (m.matches()) {
 +
 +      Calendar cal = Calendar.getInstance();
 +      cal.setTime(new Date(System.currentTimeMillis()));
 +      Integer date = Integer.parseInt(m.group(1));
 +      Integer hour = Integer.parseInt(m.group(2));
 +      Integer min = Integer.parseInt(m.group(3));
 +      Integer sec = Integer.parseInt(m.group(4));
 +      Integer ms = Integer.parseInt(m.group(5));
 +      String clazz = m.group(6);
 +      String level = m.group(7);
 +      String message = m.group(8);
 +      // Apply the regex filter if supplied
 +      if (null != lineFilter) {
 +        Matcher match = lineFilter.matcher(message);
 +        if (!match.matches())
 +          return null;
 +      }
 +      // URL encode the message
-       message = URLEncoder.encode(message, Constants.UTF8.name());
++      message = URLEncoder.encode(message, UTF_8.name());
 +      // Assume that we are processing logs from today.
 +      // If the date in the line is greater than today, then it must be
 +      // from the previous month.
 +      cal.set(Calendar.DATE, date);
 +      cal.set(Calendar.HOUR_OF_DAY, hour);
 +      cal.set(Calendar.MINUTE, min);
 +      cal.set(Calendar.SECOND, sec);
 +      cal.set(Calendar.MILLISECOND, ms);
 +      if (date > cal.get(Calendar.DAY_OF_MONTH)) {
 +        cal.add(Calendar.MONTH, -1);
 +      }
 +      long ts = cal.getTimeInMillis();
 +      // If this event is not between the start and end dates, then skip it.
 +      if (null != dateFilter && !dateFilter.containsLong(ts))
 +        return null;
 +      Category c = Logger.getLogger(clazz);
 +      Level l = Level.toLevel(level);
 +      LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
 +          (Map<?,?>) null);
 +      // Check the log level filter
 +      if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
 +        return null;
 +      }
 +      result = format(event);
 +    }
 +    return result;
 +  }
 +
 +  private static class DateConverter implements IStringConverter<Date> {
 +    @Override
 +    public Date convert(String value) {
 +      SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
 +      try {
 +        return formatter.parse(value);
 +      } catch (ParseException e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +
 +  }
 +
 +  private static class Opts extends Help {
 +
 +    @Parameter(names = {"-d", "--logDirectory"}, description = "ACCUMULO log directory path", required = true)
 +    String dir;
 +
 +    @Parameter(names = {"-f", "--fileFilter"}, description = "filter to apply to names of logs")
 +    String filter;
 +
 +    @Parameter(names = {"-h", "--host"}, description = "host where chainsaw is running", required = true)
 +    String hostname;
 +
 +    @Parameter(names = {"-p", "--port"}, description = "port where XMLSocketReceiver is listening", required = true)
 +    int portnum;
 +
 +    @Parameter(names = {"-s", "--start"}, description = "start date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
 +    Date startDate;
 +
 +    @Parameter(names = {"-e", "--end"}, description = "end date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
 +    Date endDate;
 +
 +    @Parameter(names = {"-l", "--level"}, description = "filter log level")
 +    String level;
 +
 +    @Parameter(names = {"-m", "--messageFilter"}, description = "regex filter for log messages")
 +    String regex;
 +  }
 +
 +  /**
 +   * 
 +   * @param args
 +   *          <ol>
 +   *          <li>path to log directory</li>
 +   *          <li>filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive)</li>
 +   *          <li>chainsaw host</li>
 +   *          <li>chainsaw port</li>
 +   *          <li>start date filter</li>
 +   *          <li>end date filter</li>
 +   *          <li>optional regex filter to match on each log4j message</li>
 +   *          <li>optional level filter</li>
 +   *          </ol>
 +   */
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    opts.parseArgs(SendLogToChainsaw.class.getName(), args);
 +
 +    SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
 +    c.processLogFiles();
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index b6ca527,0000000..401d98c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@@ -1,45 -1,0 +1,47 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class SystemPropUtil {
 +  public static boolean setSystemProperty(String property, String value) throws KeeperException, InterruptedException {
 +    Property p = Property.getPropertyByKey(property);
 +    if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property))
 +      return false;
 +    
 +    // create the zk node for this property and set it's data to the specified value
 +    String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
-     ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    
 +    return true;
 +  }
 +  
 +  public static void removeSystemProperty(String property) throws InterruptedException, KeeperException {
 +    String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
 +    ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.FAIL);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index bcaf9b0,0000000..6ee1593
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class TablePropUtil {
 +  public static boolean setTableProperty(String tableId, String property, String value) throws KeeperException, InterruptedException {
 +    if (!isPropertyValid(property, value))
 +      return false;
 +    
 +    // create the zk node for per-table properties for this table if it doesn't already exist
 +    String zkTablePath = getTablePath(tableId);
 +    ZooReaderWriter.getInstance().putPersistentData(zkTablePath, new byte[0], NodeExistsPolicy.SKIP);
 +    
 +    // create the zk node for this property and set it's data to the specified value
 +    String zPath = zkTablePath + "/" + property;
-     ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +    
 +    return true;
 +  }
 +  
 +  public static boolean isPropertyValid(String property, String value) {
 +    Property p = Property.getPropertyByKey(property);
 +    if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
 +      return false;
 +    
 +    return true;
 +  }
 +  
 +  public static void removeTableProperty(String tableId, String property) throws InterruptedException, KeeperException {
 +    String zPath = getTablePath(tableId) + "/" + property;
 +    ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
 +  }
 +  
 +  private static String getTablePath(String tablename) {
 +    return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tablename + Constants.ZTABLE_CONF;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 34c2151,0000000..edbb103
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,72 -1,0 +1,74 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.util;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.List;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.cli.Help;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooCache;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +
 +import com.beust.jcommander.Parameter;
 +
 +public class TabletServerLocks {
 +  
 +  static class Opts extends Help {
 +    @Parameter(names="-list")
 +    boolean list = false;
 +    @Parameter(names="-delete")
 +    String delete = null;
 +  }
 +  public static void main(String[] args) throws Exception {
 +    
 +    Instance instance = HdfsZooInstance.getInstance();
 +    String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
 +    Opts opts = new Opts();
 +    opts.parseArgs(TabletServerLocks.class.getName(), args);
 +    
 +    ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
 +    
 +    if (opts.list) {
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      
 +      List<String> tabletServers = zoo.getChildren(tserverPath);
 +      
 +      for (String tabletServer : tabletServers) {
 +        byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
 +        String holder = null;
 +        if (lockData != null) {
-           holder = new String(lockData, Constants.UTF8);
++          holder = new String(lockData, UTF_8);
 +        }
 +        
 +        System.out.printf("%32s %16s%n", tabletServer, holder);
 +      }
 +    } else if (opts.delete != null) {
 +      ZooLock.deleteLock(tserverPath + "/" + args[1]);
 +    } else {
 +      System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
 +    }
 +    
 +  }
 +  
 +}


[48/50] [abbrv] Merge branch '1.6'

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
index 339c9a8,0000000..90a8622
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@@ -1,796 -1,0 +1,796 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
 +import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
 +
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.StringTokenizer;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.ClientSideIteratorScanner;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.IsolatedScanner;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.RowIterator;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.TabletLocator;
 +import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
 +import org.apache.accumulo.core.client.mock.impl.MockTabletLocator;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.master.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.TablePermission;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.TextUtil;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.io.MapWritable;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.util.StringUtils;
 +
 +import com.google.common.collect.Maps;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class InputConfigurator extends ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link Scanner}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum ScanOpts {
 +    TABLE_NAME, AUTHORIZATIONS, RANGES, COLUMNS, ITERATORS, TABLE_CONFIGS
 +  }
 +
 +  /**
 +   * Configuration keys for various features.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum Features {
 +    AUTO_ADJUST_RANGES, SCAN_ISOLATION, USE_LOCAL_ITERATORS, SCAN_OFFLINE
 +  }
 +
 +  /**
 +   * Sets the name of the input table, over which this job will scan.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table to use when the tablename is null in the write call
 +   * @since 1.6.0
 +   */
 +  public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
 +    checkArgument(tableName != null, "tableName is null");
 +    conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
 +  }
 +
 +  /**
 +   * Sets the name of the input table, over which this job will scan.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @since 1.6.0
 +   */
 +  public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
 +  }
 +
 +  /**
 +   * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param auths
 +   *          the user's authorizations
 +   * @since 1.6.0
 +   */
 +  public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf, Authorizations auths) {
 +    if (auths != null && !auths.isEmpty())
 +      conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
 +  }
 +
 +  /**
 +   * Gets the authorizations to set for the scans from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the Accumulo scan authorizations
 +   * @since 1.6.0
 +   * @see #setScanAuthorizations(Class, Configuration, Authorizations)
 +   */
 +  public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
 +    String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
-     return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(StandardCharsets.UTF_8));
++    return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(UTF_8));
 +  }
 +
 +  /**
 +   * Sets the input ranges to scan on all input tables for this job. If not set, the entire table will be scanned.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param ranges
 +   *          the ranges that will be mapped over
 +   * @throws IllegalArgumentException
 +   *           if the ranges cannot be encoded into base 64
 +   * @since 1.6.0
 +   */
 +  public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
 +    checkArgument(ranges != null, "ranges is null");
 +
 +    ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
 +    try {
 +      for (Range r : ranges) {
 +        ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +        r.write(new DataOutputStream(baos));
 +        rangeStrings.add(Base64.encodeBase64String(baos.toByteArray()));
 +      }
 +      conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES), rangeStrings.toArray(new String[0]));
 +    } catch (IOException ex) {
 +      throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
 +    }
 +  }
 +
 +  /**
 +   * Gets the ranges to scan over from a job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the ranges
 +   * @throws IOException
 +   *           if the ranges have been encoded improperly
 +   * @since 1.6.0
 +   * @see #setRanges(Class, Configuration, Collection)
 +   */
 +  public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
 +
 +    Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
 +    List<Range> ranges = new ArrayList<Range>();
 +    for (String rangeString : encodedRanges) {
-       ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes(StandardCharsets.UTF_8)));
++      ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes(UTF_8)));
 +      Range range = new Range();
 +      range.readFields(new DataInputStream(bais));
 +      ranges.add(range);
 +    }
 +    return ranges;
 +  }
 +
 +  /**
 +   * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return a list of iterators
 +   * @since 1.6.0
 +   * @see #addIterator(Class, Configuration, IteratorSetting)
 +   */
 +  public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
 +    String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
 +
 +    // If no iterators are present, return an empty list
 +    if (iterators == null || iterators.isEmpty())
 +      return new ArrayList<IteratorSetting>();
 +
 +    // Compose the set of iterators encoded in the job configuration
 +    StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
 +    List<IteratorSetting> list = new ArrayList<IteratorSetting>();
 +    try {
 +      while (tokens.hasMoreTokens()) {
 +        String itstring = tokens.nextToken();
-         ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(StandardCharsets.UTF_8)));
++        ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(UTF_8)));
 +        list.add(new IteratorSetting(new DataInputStream(bais)));
 +        bais.close();
 +      }
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("couldn't decode iterator settings");
 +    }
 +    return list;
 +  }
 +
 +  /**
 +   * Restricts the columns that will be mapped over for the single input table on this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param columnFamilyColumnQualifierPairs
 +   *          a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
 +   *          selected. An empty set is the default and is equivalent to scanning the all columns.
 +   * @throws IllegalArgumentException
 +   *           if the column family is null
 +   * @since 1.6.0
 +   */
 +  public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
 +    checkArgument(columnFamilyColumnQualifierPairs != null, "columnFamilyColumnQualifierPairs is null");
 +    String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
 +    conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
 +  }
 +
 +  public static String[] serializeColumns(Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
 +    checkArgument(columnFamilyColumnQualifierPairs != null, "columnFamilyColumnQualifierPairs is null");
 +    ArrayList<String> columnStrings = new ArrayList<String>(columnFamilyColumnQualifierPairs.size());
 +    for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
 +
 +      if (column.getFirst() == null)
 +        throw new IllegalArgumentException("Column family can not be null");
 +
 +      String col = Base64.encodeBase64String(TextUtil.getBytes(column.getFirst()));
 +      if (column.getSecond() != null)
 +        col += ":" + Base64.encodeBase64String(TextUtil.getBytes(column.getSecond()));
 +      columnStrings.add(col);
 +    }
 +
 +    return columnStrings.toArray(new String[0]);
 +  }
 +
 +  /**
 +   * Gets the columns to be mapped over from this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return a set of columns
 +   * @since 1.6.0
 +   * @see #fetchColumns(Class, Configuration, Collection)
 +   */
 +  public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass, Configuration conf) {
 +    checkArgument(conf != null, "conf is null");
 +    String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
 +    List<String> serialized = new ArrayList<String>();
 +    if (confValue != null) {
 +      // Split and include any trailing empty strings to allow empty column families
 +      for (String val : confValue.split(",", -1)) {
 +        serialized.add(val);
 +      }
 +    }
 +    return deserializeFetchedColumns(serialized);
 +  }
 +
 +  public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
 +    Set<Pair<Text,Text>> columns = new HashSet<Pair<Text,Text>>();
 +
 +    if (null == serialized) {
 +      return columns;
 +    }
 +
 +    for (String col : serialized) {
 +      int idx = col.indexOf(":");
-       Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(StandardCharsets.UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(
-           StandardCharsets.UTF_8)));
-       Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(StandardCharsets.UTF_8)));
++      Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(
++          UTF_8)));
++      Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(UTF_8)));
 +      columns.add(new Pair<Text,Text>(cf, cq));
 +    }
 +    return columns;
 +  }
 +
 +  /**
 +   * Encode an iterator on the input for the single input table associated with this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param cfg
 +   *          the configuration of the iterator
 +   * @throws IllegalArgumentException
 +   *           if the iterator can't be serialized into the configuration
 +   * @since 1.6.0
 +   */
 +  public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    String newIter;
 +    try {
 +      cfg.write(new DataOutputStream(baos));
 +      newIter = Base64.encodeBase64String(baos.toByteArray());
 +      baos.close();
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("unable to serialize IteratorSetting");
 +    }
 +
 +    String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
 +    String iterators = conf.get(confKey);
 +    // No iterators specified yet, create a new string
 +    if (iterators == null || iterators.isEmpty()) {
 +      iterators = newIter;
 +    } else {
 +      // append the next iterator & reset
 +      iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
 +    }
 +    // Store the iterators w/ the job
 +    conf.set(confKey, iterators);
 +  }
 +
 +  /**
 +   * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
 +   * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
 +   * 
 +   * <p>
 +   * By default, this feature is <b>enabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @see #setRanges(Class, Configuration, Collection)
 +   * @since 1.6.0
 +   */
 +  public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has auto-adjust ranges enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return false if the feature is disabled, true otherwise
 +   * @since 1.6.0
 +   * @see #setAutoAdjustRanges(Class, Configuration, boolean)
 +   */
 +  public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
 +  }
 +
 +  /**
 +   * Controls the use of the {@link IsolatedScanner} in this job.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has isolation enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setScanIsolation(Class, Configuration, boolean)
 +   */
 +  public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
 +  }
 +
 +  /**
 +   * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
 +   * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration uses local iterators.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setLocalIterators(Class, Configuration, boolean)
 +   */
 +  public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
 +  }
 +
 +  /**
 +   * <p>
 +   * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
 +   * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
 +   * fail.
 +   * 
 +   * <p>
 +   * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
 +   * 
 +   * <p>
 +   * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
 +   * on the mapper's classpath.
 +   * 
 +   * <p>
 +   * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
 +   * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
 +   * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
 +   * 
 +   * <p>
 +   * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
 +   * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether a configuration has the offline table scan feature enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setOfflineTableScan(Class, Configuration, boolean)
 +   */
 +  public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
 +  }
 +
 +  /**
 +   * Sets configurations for multiple tables at a time.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param configs
 +   *          an array of {@link InputTableConfig} objects to associate with the job
 +   * @since 1.6.0
 +   */
 +  public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String,InputTableConfig> configs) {
 +    MapWritable mapWritable = new MapWritable();
 +    for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
 +      mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
 +
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    try {
 +      mapWritable.write(new DataOutputStream(baos));
 +    } catch (IOException e) {
 +      throw new IllegalStateException("Table configuration could not be serialized.");
 +    }
 +
 +    String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
 +    conf.set(confKey, Base64.encodeBase64String(baos.toByteArray()));
 +  }
 +
 +  /**
 +   * Returns all {@link InputTableConfig} objects associated with this job.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return all of the table query configs for the job
 +   * @since 1.6.0
 +   */
 +  public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
 +    Map<String,InputTableConfig> configs = new HashMap<String,InputTableConfig>();
 +    Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
 +    if (defaultConfig != null)
 +      configs.put(defaultConfig.getKey(), defaultConfig.getValue());
 +    String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
 +    MapWritable mapWritable = new MapWritable();
 +    if (configString != null) {
 +      try {
-         byte[] bytes = Base64.decodeBase64(configString.getBytes(StandardCharsets.UTF_8));
++        byte[] bytes = Base64.decodeBase64(configString.getBytes(UTF_8));
 +        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
 +        mapWritable.readFields(new DataInputStream(bais));
 +        bais.close();
 +      } catch (IOException e) {
 +        throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
 +      }
 +    }
 +    for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
 +      configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());
 +
 +    return configs;
 +  }
 +
 +  /**
 +   * Returns the {@link InputTableConfig} for the given table
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table name for which to fetch the table query config
 +   * @return the table query config for the given table name (if it exists) and null if it does not
 +   * @since 1.6.0
 +   */
 +  public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf, String tableName) {
 +    Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
 +    return queryConfigs.get(tableName);
 +  }
 +
 +  /**
 +   * Initializes an Accumulo {@link TabletLocator} based on the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableId
 +   *          The table id for which to initialize the {@link TabletLocator}
 +   * @return an Accumulo tablet locator
 +   * @throws TableNotFoundException
 +   *           if the table name set on the configuration doesn't exist
 +   * @since 1.6.0
 +   */
 +  public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf, String tableId) throws TableNotFoundException {
 +    String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
 +    if ("MockInstance".equals(instanceType))
 +      return new MockTabletLocator();
 +    Instance instance = getInstance(implementingClass, conf);
 +    return TabletLocator.getLocator(instance, new Text(tableId));
 +  }
 +
 +  // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
 +  /**
 +   * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @throws IOException
 +   *           if the context is improperly configured
 +   * @since 1.6.0
 +   */
 +  public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
 +
 +    Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
 +    if (!isConnectorInfoSet(implementingClass, conf))
 +      throw new IOException("Input info has not been set.");
 +    String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
 +    if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
 +      throw new IOException("Instance info has not been set.");
 +    // validate that we can connect as configured
 +    try {
 +      String principal = getPrincipal(implementingClass, conf);
 +      AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
 +      Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
 +      if (!c.securityOperations().authenticateUser(principal, token))
 +        throw new IOException("Unable to authenticate user");
 +
 +      if (getInputTableConfigs(implementingClass, conf).size() == 0)
 +        throw new IOException("No table set.");
 +
 +      for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
 +        if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
 +          throw new IOException("Unable to access table");
 +      }
 +      for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
 +        InputTableConfig tableConfig = tableConfigEntry.getValue();
 +        if (!tableConfig.shouldUseLocalIterators()) {
 +          if (tableConfig.getIterators() != null) {
 +            for (IteratorSetting iter : tableConfig.getIterators()) {
 +              if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
 +                throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
 +            }
 +          }
 +        }
 +      }
 +    } catch (AccumuloException e) {
 +      throw new IOException(e);
 +    } catch (AccumuloSecurityException e) {
 +      throw new IOException(e);
 +    } catch (TableNotFoundException e) {
 +      throw new IOException(e);
 +    }
 +  }
 +
 +  /**
 +   * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the configuration based on the properties set using the single-table
 +   * input methods.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop instance for which to retrieve the configuration
 +   * @return the config object built from the single input table properties set on the job
 +   * @since 1.6.0
 +   */
 +  protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(Class<?> implementingClass, Configuration conf) {
 +    String tableName = getInputTableName(implementingClass, conf);
 +    if (tableName != null) {
 +      InputTableConfig queryConfig = new InputTableConfig();
 +      List<IteratorSetting> itrs = getIterators(implementingClass, conf);
 +      if (itrs != null)
 +        queryConfig.setIterators(itrs);
 +      Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
 +      if (columns != null)
 +        queryConfig.fetchColumns(columns);
 +      List<Range> ranges = null;
 +      try {
 +        ranges = getRanges(implementingClass, conf);
 +      } catch (IOException e) {
 +        throw new RuntimeException(e);
 +      }
 +      if (ranges != null)
 +        queryConfig.setRanges(ranges);
 +
 +      queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)).setUseIsolatedScanners(isIsolated(implementingClass, conf))
 +          .setUseLocalIterators(usesLocalIterators(implementingClass, conf)).setOfflineScan(isOfflineScan(implementingClass, conf));
 +      return Maps.immutableEntry(tableName, queryConfig);
 +    }
 +    return null;
 +  }
 +
 +  public static Map<String,Map<KeyExtent,List<Range>>> binOffline(String tableId, List<Range> ranges, Instance instance, Connector conn)
 +      throws AccumuloException, TableNotFoundException {
 +    Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
 +
 +    if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
 +      Tables.clearCache(instance);
 +      if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
 +        throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
 +      }
 +    }
 +
 +    for (Range range : ranges) {
 +      Text startRow;
 +
 +      if (range.getStartKey() != null)
 +        startRow = range.getStartKey().getRow();
 +      else
 +        startRow = new Text();
 +
 +      Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
 +      Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
 +      MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
 +      scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
 +      scanner.setRange(metadataRange);
 +
 +      RowIterator rowIter = new RowIterator(scanner);
 +      KeyExtent lastExtent = null;
 +      while (rowIter.hasNext()) {
 +        Iterator<Map.Entry<Key,Value>> row = rowIter.next();
 +        String last = "";
 +        KeyExtent extent = null;
 +        String location = null;
 +
 +        while (row.hasNext()) {
 +          Map.Entry<Key,Value> entry = row.next();
 +          Key key = entry.getKey();
 +
 +          if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
 +            last = entry.getValue().toString();
 +          }
 +
 +          if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
 +              || key.getColumnFamily().equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
 +            location = entry.getValue().toString();
 +          }
 +
 +          if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
 +            extent = new KeyExtent(key.getRow(), entry.getValue());
 +          }
 +
 +        }
 +
 +        if (location != null)
 +          return null;
 +
 +        if (!extent.getTableId().toString().equals(tableId)) {
 +          throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
 +        }
 +
 +        if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
 +          throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
 +        }
 +
 +        Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
 +        if (tabletRanges == null) {
 +          tabletRanges = new HashMap<KeyExtent,List<Range>>();
 +          binnedRanges.put(last, tabletRanges);
 +        }
 +
 +        List<Range> rangeList = tabletRanges.get(extent);
 +        if (rangeList == null) {
 +          rangeList = new ArrayList<Range>();
 +          tabletRanges.put(extent, rangeList);
 +        }
 +
 +        rangeList.add(range);
 +
 +        if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
 +          break;
 +        }
 +
 +        lastExtent = extent;
 +      }
 +
 +    }
 +    return binnedRanges;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
index 727971a,0000000..13b67d5
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
@@@ -1,204 -1,0 +1,204 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.impl;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataInputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
- import java.nio.charset.StandardCharsets;
- 
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.hadoop.conf.Configuration;
 +
 +/**
 + * @since 1.6.0
 + */
 +public class OutputConfigurator extends ConfiguratorBase {
 +
 +  /**
 +   * Configuration keys for {@link BatchWriter}.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum WriteOpts {
 +    DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
 +  }
 +
 +  /**
 +   * Configuration keys for various features.
 +   * 
 +   * @since 1.6.0
 +   */
 +  public static enum Features {
 +    CAN_CREATE_TABLES, SIMULATION_MODE
 +  }
 +
 +  /**
 +   * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
 +   * underscores.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param tableName
 +   *          the table to use when the tablename is null in the write call
 +   * @since 1.6.0
 +   */
 +  public static void setDefaultTableName(Class<?> implementingClass, Configuration conf, String tableName) {
 +    if (tableName != null)
 +      conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
 +  }
 +
 +  /**
 +   * Gets the default table name from the configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the default table name
 +   * @since 1.6.0
 +   * @see #setDefaultTableName(Class, Configuration, String)
 +   */
 +  public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
 +    return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
 +  }
 +
 +  /**
 +   * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
 +   * used. Setting the configuration multiple times overwrites any previous configuration.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param bwConfig
 +   *          the configuration for the {@link BatchWriter}
 +   * @since 1.6.0
 +   */
 +  public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf, BatchWriterConfig bwConfig) {
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    String serialized;
 +    try {
 +      bwConfig.write(new DataOutputStream(baos));
-       serialized = new String(baos.toByteArray(), StandardCharsets.UTF_8);
++      serialized = new String(baos.toByteArray(), UTF_8);
 +      baos.close();
 +    } catch (IOException e) {
 +      throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
 +    }
 +    conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
 +  }
 +
 +  /**
 +   * Gets the {@link BatchWriterConfig} settings.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return the configuration object
 +   * @since 1.6.0
 +   * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
 +   */
 +  public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
 +    String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
 +    BatchWriterConfig bwConfig = new BatchWriterConfig();
 +    if (serialized == null || serialized.isEmpty()) {
 +      return bwConfig;
 +    } else {
 +      try {
-         ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(StandardCharsets.UTF_8));
++        ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
 +        bwConfig.readFields(new DataInputStream(bais));
 +        bais.close();
 +        return bwConfig;
 +      } catch (IOException e) {
 +        throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setCreateTables(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether tables are permitted to be created as needed.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is disabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setCreateTables(Class, Configuration, boolean)
 +   */
 +  public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
 +  }
 +
 +  /**
 +   * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
 +   * 
 +   * <p>
 +   * By default, this feature is <b>disabled</b>.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @param enableFeature
 +   *          the feature is enabled if true, disabled otherwise
 +   * @since 1.6.0
 +   */
 +  public static void setSimulationMode(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
 +    conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
 +  }
 +
 +  /**
 +   * Determines whether this feature is enabled.
 +   * 
 +   * @param implementingClass
 +   *          the class whose name will be used as a prefix for the property configuration key
 +   * @param conf
 +   *          the Hadoop configuration object to configure
 +   * @return true if the feature is enabled, false otherwise
 +   * @since 1.6.0
 +   * @see #setSimulationMode(Class, Configuration, boolean)
 +   */
 +  public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
 +    return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 2950091,0000000..c0c0097
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@@ -1,135 -1,0 +1,136 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapreduce.lib.partition;
 +
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.io.BufferedReader;
 +import java.io.FileInputStream;
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.InputStreamReader;
 +import java.net.URI;
- import java.nio.charset.StandardCharsets;
 +import java.util.Arrays;
 +import java.util.Scanner;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.client.mapreduce.lib.impl.DistributedCacheHelper;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.hadoop.conf.Configurable;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.io.Writable;
 +import org.apache.hadoop.mapreduce.Job;
 +import org.apache.hadoop.mapreduce.Partitioner;
 +
 +/**
 + * Hadoop partitioner that uses ranges, and optionally sub-bins based on hashing.
 + */
 +public class RangePartitioner extends Partitioner<Text,Writable> implements Configurable {
 +  private static final String PREFIX = RangePartitioner.class.getName();
 +  private static final String CUTFILE_KEY = PREFIX + ".cutFile";
 +  private static final String NUM_SUBBINS = PREFIX + ".subBins";
 +
 +  private Configuration conf;
 +
 +  @Override
 +  public int getPartition(Text key, Writable value, int numPartitions) {
 +    try {
 +      return findPartition(key, getCutPoints(), getNumSubBins());
 +    } catch (IOException e) {
 +      throw new RuntimeException(e);
 +    }
 +  }
 +
 +  int findPartition(Text key, Text[] array, int numSubBins) {
 +    // find the bin for the range, and guarantee it is positive
 +    int index = Arrays.binarySearch(array, key);
 +    index = index < 0 ? (index + 1) * -1 : index;
 +
 +    // both conditions work with numSubBins == 1, but this check is to avoid
 +    // hashing, when we don't need to, for speed
 +    if (numSubBins < 2)
 +      return index;
 +    return (key.toString().hashCode() & Integer.MAX_VALUE) % numSubBins + index * numSubBins;
 +  }
 +
 +  private int _numSubBins = 0;
 +
 +  private synchronized int getNumSubBins() {
 +    if (_numSubBins < 1) {
 +      // get number of sub-bins and guarantee it is positive
 +      _numSubBins = Math.max(1, getConf().getInt(NUM_SUBBINS, 1));
 +    }
 +    return _numSubBins;
 +  }
 +
 +  private Text cutPointArray[] = null;
 +
 +  private synchronized Text[] getCutPoints() throws IOException {
 +    if (cutPointArray == null) {
 +      String cutFileName = conf.get(CUTFILE_KEY);
 +      Path[] cf = DistributedCacheHelper.getLocalCacheFiles(conf);
 +
 +      if (cf != null) {
 +        for (Path path : cf) {
 +          if (path.toUri().getPath().endsWith(cutFileName.substring(cutFileName.lastIndexOf('/')))) {
 +            TreeSet<Text> cutPoints = new TreeSet<Text>();
-             Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()), StandardCharsets.UTF_8)));
++            Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()), UTF_8)));
 +            try {
 +              while (in.hasNextLine())
-                 cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(StandardCharsets.UTF_8))));
++                cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(UTF_8))));
 +            } finally {
 +              in.close();
 +            }
 +            cutPointArray = cutPoints.toArray(new Text[cutPoints.size()]);
 +            break;
 +          }
 +        }
 +      }
 +      if (cutPointArray == null)
 +        throw new FileNotFoundException(cutFileName + " not found in distributed cache");
 +    }
 +    return cutPointArray;
 +  }
 +
 +  @Override
 +  public Configuration getConf() {
 +    return conf;
 +  }
 +
 +  @Override
 +  public void setConf(Configuration conf) {
 +    this.conf = conf;
 +  }
 +
 +  /**
 +   * Sets the hdfs file name to use, containing a newline separated list of Base64 encoded split points that represent ranges for partitioning
 +   */
 +  public static void setSplitFile(Job job, String file) {
 +    URI uri = new Path(file).toUri();
 +    DistributedCacheHelper.addCacheFile(uri, job.getConfiguration());
 +    job.getConfiguration().set(CUTFILE_KEY, uri.getPath());
 +  }
 +
 +  /**
 +   * Sets the number of random sub-bins per range
 +   */
 +  public static void setNumSubBins(Job job, int num) {
 +    job.getConfiguration().setInt(NUM_SUBBINS, num);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index 0febb58,8216441..aa10276
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.minicluster.impl;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.BufferedReader;
  import java.io.BufferedWriter;
  import java.io.File;
@@@ -105,8 -108,6 +107,7 @@@ import org.apache.zookeeper.server.ZooK
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
- import com.google.common.base.Charsets;
 +import com.google.common.base.Joiner;
  import com.google.common.base.Predicate;
  import com.google.common.collect.Maps;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
index a778add,30da0cba..bd0782d
--- a/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.proxy;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
index 5f84c2f,ec1c9d2..4850c19
--- a/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.proxy;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.Collections;
  import java.util.Date;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/proxy/src/main/java/org/apache/accumulo/proxy/Util.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/Util.java
index 94b0709,7333539..bc0db72
--- a/proxy/src/main/java/org/apache/accumulo/proxy/Util.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/Util.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.proxy;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.math.BigInteger;
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.Random;
  
  import org.apache.accumulo.proxy.thrift.IteratorSetting;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index e4d6264,6c40094..5c93a53
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 7169215,05f3c6c..efbd882
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -16,9 -16,10 +16,10 @@@
   */
  package org.apache.accumulo.server.client;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.Collections;
  import java.util.List;
  import java.util.UUID;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
index a154fdc,18feb4f..2ad1c3d
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.conf;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.List;
  import java.util.Map;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 0e9ee95,6d25f1d..c472581
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.conf;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.HashMap;
  import java.util.List;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 9d56265,587e38a..e4f61f9
--- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.constraints;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 1fd9aa3,7c4b8d3..45e883d
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@@ -16,15 -16,14 +16,15 @@@
   */
  package org.apache.accumulo.server.init;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
  import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
  import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
  
  import java.io.FileNotFoundException;
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.Arrays;
 +import java.util.Collections;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Locale;
@@@ -412,8 -404,7 +412,8 @@@ public class Initialize 
      zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
      zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
      zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
-     zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(StandardCharsets.UTF_8),
 -    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8), NodeExistsPolicy.FAIL);
++    zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8),
 +        NodeExistsPolicy.FAIL);
      zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
      zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
      zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index 4a66db7,002389a..e577153
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@@ -16,6 -16,7 +16,7 @@@
   */
  package org.apache.accumulo.server.master;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
  import static org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP;
  
  import java.nio.ByteBuffer;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
index 0e1f5ee,568e6f6..3f7f167
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.master.state;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.ArrayList;
  import java.util.List;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
index 8a09e75,b4ef519..f473ba3
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
@@@ -16,12 -16,13 +16,12 @@@
   */
  package org.apache.accumulo.server.master.state;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.io.ObjectInputStream;
  import java.io.ObjectOutputStream;
  import java.io.Serializable;
- import java.nio.charset.StandardCharsets;
--
  import org.apache.accumulo.core.data.Mutation;
  import org.apache.accumulo.core.data.Value;
  import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
@@@ -127,8 -120,8 +127,8 @@@ public class TServerInstance implement
      return new Text(this.getSession());
    }
    
 -  public Value asMutationValue() {
 +  private Value asMutationValue() {
-     return new Value(getLocation().toString().getBytes(StandardCharsets.UTF_8));
+     return new Value(getLocation().toString().getBytes(UTF_8));
    }
    
    public HostAndPort getLocation() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
index e06b58e,91be9c3..2a84e70
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.master.state;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
index 6ab4683,8f0c9c7..1bcf482
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.master.state;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.List;
  
  import org.apache.accumulo.core.zookeeper.ZooUtil;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
index e7f06dc,15ade65..5481531
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.master.state;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
- import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Iterator;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 05ede64,409fa62..54ca8de
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.metrics;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.File;
  import java.io.FileOutputStream;
  import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
index 902c063,4e909c0..930b634
--- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.monitor;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.IOException;
  import java.net.ServerSocket;
  import java.net.Socket;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
index 590cbe4,0000000..8e9612a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
@@@ -1,274 -1,0 +1,275 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.server.replication;
 +
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.NoSuchElementException;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchScanner;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.TableOperations;
 +import org.apache.accumulo.core.client.replication.ReplicaSystem;
 +import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.replication.ReplicationConstants;
 +import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
 +import org.apache.accumulo.core.replication.ReplicationTarget;
 +import org.apache.accumulo.core.replication.StatusUtil;
 +import org.apache.accumulo.core.replication.proto.Replication.Status;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.hadoop.io.Text;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.collect.Iterables;
 +import com.google.common.collect.Maps;
 +import com.google.protobuf.InvalidProtocolBufferException;
 +
 +public class ReplicationUtil {
 +  private static final Logger log = LoggerFactory.getLogger(ReplicationUtil.class);
 +
 +  private static final String REPLICATION_TARGET_PREFIX = Property.TABLE_REPLICATION_TARGET.getKey();
 +
 +  private ZooCache zooCache;
 +
 +  public ReplicationUtil() {
 +    this(new ZooCache());
 +  }
 +
 +  public ReplicationUtil(ZooCache cache) {
 +    this.zooCache = cache;
 +  }
 +
 +  public int getMaxReplicationThreads(Map<String,String> systemProperties, MasterMonitorInfo mmi) {
 +    int activeTservers = mmi.getTServerInfoSize();
 +
 +    // The number of threads each tserver will use at most to replicate data
 +    int replicationThreadsPerServer = Integer.parseInt(systemProperties.get(Property.REPLICATION_WORKER_THREADS.getKey()));
 +
 +    // The total number of "slots" we have to replicate data
 +    return activeTservers * replicationThreadsPerServer;
 +  }
 +
 +  public int getMaxReplicationThreads(Connector conn, MasterMonitorInfo mmi) throws AccumuloException, AccumuloSecurityException {
 +    return getMaxReplicationThreads(conn.instanceOperations().getSystemConfiguration(), mmi);
 +  }
 +
 +  /**
 +   * Extract replication peers from system configuration
 +   * @param systemProperties System properties, typically from Connector.instanceOperations().getSystemConfiguration()
 +   * @return Configured replication peers
 +   */
 +  public Map<String,String> getPeers(Map<String,String> systemProperties) {
 +    Map<String,String> peers = new HashMap<>();
 +    String definedPeersPrefix = Property.REPLICATION_PEERS.getKey();
 +
 +    // Get the defined peers and what ReplicaSystem impl they're using
 +    for (Entry<String,String> property : systemProperties.entrySet()) {
 +      String key = property.getKey();
 +      // Filter out cruft that we don't want
 +      if (key.startsWith(definedPeersPrefix) && !key.startsWith(Property.REPLICATION_PEER_USER.getKey()) && !key.startsWith(Property.REPLICATION_PEER_PASSWORD.getKey())) {
 +        String peerName = property.getKey().substring(definedPeersPrefix.length());
 +        ReplicaSystem replica;
 +        try {
 +         replica = ReplicaSystemFactory.get(property.getValue());
 +        } catch (Exception e) {
 +          log.warn("Could not instantiate ReplicaSystem for {} with configuration {}", property.getKey(), property.getValue(), e);
 +          continue;
 +        }
 +
 +        peers.put(peerName, replica.getClass().getName());
 +      }
 +    }
 +
 +    return peers;
 +  }
 +
 +  public Set<ReplicationTarget> getReplicationTargets(TableOperations tops) {
 +    // The total set of configured targets
 +    final Set<ReplicationTarget> allConfiguredTargets = new HashSet<>();
 +    final Map<String,String> tableNameToId = tops.tableIdMap();
 +
 +    for (String table : tops.list()) {
 +      if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) {
 +        continue;
 +      }
 +      String localId = tableNameToId.get(table);
 +      if (null == localId) {
 +        log.trace("Could not determine ID for {}", table);
 +        continue;
 +      }
 +
 +      Iterable<Entry<String,String>> propertiesForTable;
 +      try {
 +        propertiesForTable = tops.getProperties(table);
 +      } catch (AccumuloException e) {
 +        log.debug("Could not fetch properties for " + table, e);
 +        continue;
 +      } catch (TableNotFoundException e) {
 +        log.debug("Could not fetch properties for " + table, e);
 +        continue;
 +      }
 +
 +      for (Entry<String,String> prop : propertiesForTable) {
 +        if (prop.getKey().startsWith(REPLICATION_TARGET_PREFIX)) {
 +          String peerName = prop.getKey().substring(REPLICATION_TARGET_PREFIX.length());
 +          String remoteIdentifier = prop.getValue();
 +          ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId);
 +
 +          allConfiguredTargets.add(target);
 +        }
 +      }
 +    }
 +
 +    return allConfiguredTargets;
 +  }
 +
 +  public Map<ReplicationTarget,Long> getPendingReplications(Connector conn) {
 +    final Map<ReplicationTarget,Long> counts = new HashMap<>();
 +
 +    // Read over the queued work
 +    BatchScanner bs;
 +    try {
 +      bs = conn.createBatchScanner(ReplicationConstants.TABLE_NAME, Authorizations.EMPTY, 4);
 +    } catch (TableNotFoundException e) {
 +      log.debug("No replication table exists", e);
 +      return counts;
 +    }
 +
 +    bs.setRanges(Collections.singleton(new Range()));
 +    WorkSection.limit(bs);
 +    try {
 +      Text buffer = new Text();
 +      for (Entry<Key,Value> entry : bs) {
 +        Key k = entry.getKey();
 +        k.getColumnQualifier(buffer);
 +        ReplicationTarget target = ReplicationTarget.from(buffer);
 +
 +        // TODO ACCUMULO-2835 once explicit lengths are tracked, we can give size-based estimates instead of just file-based
 +        Long count = counts.get(target);
 +        if (null == count) {
 +          counts.put(target, Long.valueOf(1l));
 +        } else {
 +          counts.put(target, count + 1);
 +        }
 +      }
 +    } finally {
 +      bs.close();
 +    }
 +
 +    return counts;
 +  }
 +
 +  /**
 +   * Fetches the absolute path of the file to be replicated.
 +   * @param conn Accumulo Connector
 +   * @param workQueuePath Root path for the Replication WorkQueue
 +   * @param queueKey The Replication work queue key
 +   * @return The absolute path for the file, or null if the key is no longer in ZooKeeper
 +   */
 +  public String getAbsolutePath(Connector conn, String workQueuePath, String queueKey) {
 +    byte[] data = zooCache.get(workQueuePath + "/" + queueKey);
 +    if (null != data) {
-       return new String(data, StandardCharsets.UTF_8);
++      return new String(data, UTF_8);
 +    }
 +
 +    return null;
 +  }
 +
 +  /**
 +   * Compute a progress string for the replication of the given WAL
 +   * @param conn Accumulo Connector
 +   * @param path Absolute path to a WAL, or null
 +   * @param target ReplicationTarget the WAL is being replicated to
 +   * @return A status message for a file being replicated
 +   */
 +  public String getProgress(Connector conn, String path, ReplicationTarget target) {
 +    // We could try to grep over the table, but without knowing the full file path, we
 +    // can't find the status quickly
 +    String status = "Unknown";
 +    if (null != path) {
 +      Scanner s;
 +      try {
 +        s = conn.createScanner(ReplicationConstants.TABLE_NAME, Authorizations.EMPTY);
 +      } catch (TableNotFoundException e) {
 +        log.debug("Replication table no long exists", e);
 +        return status;
 +      }
 +
 +      s.setRange(Range.exact(path));
 +      s.fetchColumn(WorkSection.NAME, target.toText());
 +
 +      // Fetch the work entry for this item
 +      Entry<Key,Value> kv = null;
 +      try {
 +        kv = Iterables.getOnlyElement(s);
 +      } catch (NoSuchElementException e) {
 +       log.trace("Could not find status of {} replicating to {}", path, target);
 +       status = "Unknown";
 +      } finally {
 +        s.close();
 +      }
 +
 +      // If we found the work entry for it, try to compute some progress
 +      if (null != kv) {
 +        try {
 +          Status stat = Status.parseFrom(kv.getValue().get());
 +          if (StatusUtil.isFullyReplicated(stat)) {
 +            status = "Finished";
 +          } else {
 +            if (stat.getInfiniteEnd()) {
 +              status = stat.getBegin() + "/&infin; records";
 +            } else {
 +              status = stat.getBegin() + "/" + stat.getEnd() + " records";
 +            }
 +          }
 +        } catch (InvalidProtocolBufferException e) {
 +          log.warn("Could not deserialize protobuf for {}", kv.getKey(), e);
 +          status = "Unknown";
 +        }
 +      }
 +    }
 +
 +    return status;
 +  }
 +
 +  public Map<String,String> invert(Map<String,String> map) {
 +    Map<String,String> newMap = Maps.newHashMapWithExpectedSize(map.size());
 +    for(Entry<String,String> entry : map.entrySet()) {
 +      newMap.put(entry.getValue(), entry.getKey());
 +    }
 +    return newMap;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
index 08407e4,258080c..fa90b3e
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 7989447,872fe11..33a5158
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@@ -16,6 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.security;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.io.ByteArrayOutputStream;
  import java.io.DataOutputStream;
  import java.io.IOException;
@@@ -117,14 -118,14 +118,14 @@@ public final class SystemCredentials ex
        }
  
        // seed the config with the version and instance id, so at least it's not empty
-       md.update(ServerConstants.WIRE_VERSION.toString().getBytes(StandardCharsets.UTF_8));
+       md.update(ServerConstants.WIRE_VERSION.toString().getBytes(UTF_8));
        md.update(instanceIdBytes);
  
 -      for (Entry<String,String> entry : ServerConfiguration.getSiteConfiguration()) {
 +      for (Entry<String,String> entry : SiteConfiguration.getInstance()) {
          // only include instance properties
          if (entry.getKey().startsWith(Property.INSTANCE_PREFIX.toString())) {
-           md.update(entry.getKey().getBytes(StandardCharsets.UTF_8));
-           md.update(entry.getValue().getBytes(StandardCharsets.UTF_8));
+           md.update(entry.getKey().getBytes(UTF_8));
+           md.update(entry.getValue().getBytes(UTF_8));
          }
        }
        confChecksum = md.digest();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
index d46d623,fa1632c..afe21f2
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.security.handler;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.HashSet;
  import java.util.Set;
  import java.util.TreeSet;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index bf22d1a,78597ad..0a1bd6c
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -16,8 -16,9 +16,9 @@@
   */
  package org.apache.accumulo.server.security.handler;
  
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 383c34e,41f880d..c03c40b
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.security.handler;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Collections;
  import java.util.HashMap;
  import java.util.Map;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 7d27909,78f8c7b..886c28f
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.tables;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.security.SecurityPermission;
  import java.util.Collections;
  import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 38a96d4,e10c0c1..8bf4dc8
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.tablets;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.Random;
  
  import org.apache.accumulo.core.Constants;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index 7a77879,198b2d6..fb4cba0
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -16,7 -16,8 +16,8 @@@
   */
  package org.apache.accumulo.server.util;
  
- import java.nio.charset.StandardCharsets;
 -import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ 
  import java.util.HashSet;
  import java.util.Map.Entry;
  import java.util.Set;


[08/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index b1bbb67,0000000..4f381ef
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@@ -1,112 -1,0 +1,114 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.NamespaceNotFoundException;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +
 +public class RenameTable extends MasterRepo {
 +
 +  private static final long serialVersionUID = 1L;
 +  private String tableId;
 +  private String oldTableName;
 +  private String newTableName;
 +  private String namespaceId;
 +
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.RENAME) + Utils.reserveTable(tableId, tid, true, true, TableOperation.RENAME);
 +  }
 +
 +  public RenameTable(String tableId, String oldTableName, String newTableName) throws NamespaceNotFoundException {
 +    this.tableId = tableId;
 +    this.oldTableName = oldTableName;
 +    this.newTableName = newTableName;
 +    Instance inst = HdfsZooInstance.getInstance();
 +    this.namespaceId = Tables.getNamespaceId(inst, tableId);
 +  }
 +
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +
 +    Instance instance = master.getInstance();
 +    Pair<String,String> qualifiedOldTableName = Tables.qualify(oldTableName);
 +    Pair<String,String> qualifiedNewTableName = Tables.qualify(newTableName);
 +
 +    // ensure no attempt is made to rename across namespaces
 +    if (newTableName.contains(".") && !namespaceId.equals(Namespaces.getNamespaceId(instance, qualifiedNewTableName.getFirst())))
 +      throw new ThriftTableOperationException(tableId, oldTableName, TableOperation.RENAME, TableOperationExceptionType.INVALID_NAME,
 +          "Namespace in new table name does not match the old table name");
 +
 +    IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +
 +    Utils.tableNameLock.lock();
 +    try {
 +      Utils.checkTableDoesNotExist(instance, newTableName, tableId, TableOperation.RENAME);
 +
 +      final String newName = qualifiedNewTableName.getSecond();
 +      final String oldName = qualifiedOldTableName.getSecond();
 +
 +      final String tap = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME;
 +
 +      zoo.mutate(tap, null, null, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] current) throws Exception {
-           final String currentName = new String(current, Constants.UTF8);
++          final String currentName = new String(current, UTF_8);
 +          if (currentName.equals(newName))
 +            return null; // assume in this case the operation is running again, so we are done
 +          if (!currentName.equals(oldName)) {
 +            throw new ThriftTableOperationException(null, oldTableName, TableOperation.RENAME, TableOperationExceptionType.NOTFOUND,
 +                "Name changed while processing");
 +          }
-           return newName.getBytes(Constants.UTF8);
++          return newName.getBytes(UTF_8);
 +        }
 +      });
 +      Tables.clearCache(instance);
 +    } finally {
 +      Utils.tableNameLock.unlock();
 +      Utils.unreserveTable(tableId, tid, true);
 +      Utils.unreserveNamespace(this.namespaceId, tid, false);
 +    }
 +
 +    Logger.getLogger(RenameTable.class).debug("Renamed table " + tableId + " " + oldTableName + " " + newTableName);
 +
 +    return null;
 +  }
 +
 +  @Override
 +  public void undo(long tid, Master env) throws Exception {
 +    Utils.unreserveTable(tableId, tid, true);
 +    Utils.unreserveNamespace(namespaceId, tid, false);
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 7ede305,0000000..3e8704e
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -1,163 -1,0 +1,165 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tableOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.math.BigInteger;
 +import java.util.concurrent.locks.Lock;
 +import java.util.concurrent.locks.ReentrantLock;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.Namespaces;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperation;
 +import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.util.Base64;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
 +import org.apache.accumulo.fate.zookeeper.ZooReservation;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.zookeeper.ZooQueueLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +public class Utils {
 +  private static final byte[] ZERO_BYTE = new byte[] {'0'};
 +
 +  static void checkTableDoesNotExist(Instance instance, String tableName, String tableId, TableOperation operation) throws ThriftTableOperationException {
 +
 +    String id = Tables.getNameToIdMap(instance).get(tableName);
 +
 +    if (id != null && !id.equals(tableId))
 +      throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.EXISTS, null);
 +  }
 +
 +  static String getNextTableId(String tableName, Instance instance) throws ThriftTableOperationException {
 +
 +    String tableId = null;
 +    try {
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +      final String ntp = ZooUtil.getRoot(instance) + Constants.ZTABLES;
 +      byte[] nid = zoo.mutate(ntp, ZERO_BYTE, ZooUtil.PUBLIC, new Mutator() {
 +        @Override
 +        public byte[] mutate(byte[] currentValue) throws Exception {
-           BigInteger nextId = new BigInteger(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++          BigInteger nextId = new BigInteger(new String(currentValue, UTF_8), Character.MAX_RADIX);
 +          nextId = nextId.add(BigInteger.ONE);
-           return nextId.toString(Character.MAX_RADIX).getBytes(Constants.UTF8);
++          return nextId.toString(Character.MAX_RADIX).getBytes(UTF_8);
 +        }
 +      });
-       return new String(nid, Constants.UTF8);
++      return new String(nid, UTF_8);
 +    } catch (Exception e1) {
 +      Logger.getLogger(CreateTable.class).error("Failed to assign tableId to " + tableName, e1);
 +      throw new ThriftTableOperationException(tableId, tableName, TableOperation.CREATE, TableOperationExceptionType.OTHER, e1.getMessage());
 +    }
 +  }
 +
 +  static final Lock tableNameLock = new ReentrantLock();
 +  static final Lock idLock = new ReentrantLock();
 +  private static final Logger log = Logger.getLogger(Utils.class);
 +
 +  public static long reserveTable(String tableId, long tid, boolean writeLock, boolean tableMustExist, TableOperation op) throws Exception {
 +    if (getLock(tableId, tid, writeLock).tryLock()) {
 +      if (tableMustExist) {
 +        Instance instance = HdfsZooInstance.getInstance();
 +        IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +        if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId))
 +          throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist");
 +      }
 +      log.info("table " + tableId + " (" + Long.toHexString(tid) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
 +      return 0;
 +    } else
 +      return 100;
 +  }
 +
 +  public static void unreserveTable(String tableId, long tid, boolean writeLock) throws Exception {
 +    getLock(tableId, tid, writeLock).unlock();
 +    log.info("table " + tableId + " (" + Long.toHexString(tid) + ") unlocked for " + (writeLock ? "write" : "read"));
 +  }
 +
 +  public static void unreserveNamespace(String namespaceId, long id, boolean writeLock) throws Exception {
 +    getLock(namespaceId, id, writeLock).unlock();
 +    log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") unlocked for " + (writeLock ? "write" : "read"));
 +  }
 +
 +  public static long reserveNamespace(String namespaceId, long id, boolean writeLock, boolean mustExist, TableOperation op) throws Exception {
 +    if (getLock(namespaceId, id, writeLock).tryLock()) {
 +      if (mustExist) {
 +        Instance instance = HdfsZooInstance.getInstance();
 +        IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +        if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId))
 +          throw new ThriftTableOperationException(namespaceId, "", op, TableOperationExceptionType.NAMESPACE_NOTFOUND, "Namespace does not exist");
 +      }
 +      log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
 +      return 0;
 +    } else
 +      return 100;
 +  }
 +
 +  public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
 +    Instance instance = HdfsZooInstance.getInstance();
 +
 +    String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
-         + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++        + Base64.encodeBase64String(directory.getBytes(UTF_8));
 +
 +    IZooReaderWriter zk = ZooReaderWriter.getInstance();
 +
 +    if (ZooReservation.attempt(zk, resvPath, String.format("%016x", tid), "")) {
 +      return 0;
 +    } else
 +      return 50;
 +  }
 +
 +  public static void unreserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
 +    Instance instance = HdfsZooInstance.getInstance();
 +    String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
-         + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++        + Base64.encodeBase64String(directory.getBytes(UTF_8));
 +    ZooReservation.release(ZooReaderWriter.getInstance(), resvPath, String.format("%016x", tid));
 +  }
 +
 +  private static Lock getLock(String tableId, long tid, boolean writeLock) throws Exception {
-     byte[] lockData = String.format("%016x", tid).getBytes(Constants.UTF8);
++    byte[] lockData = String.format("%016x", tid).getBytes(UTF_8);
 +    ZooQueueLock qlock = new ZooQueueLock(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLE_LOCKS + "/" + tableId, false);
 +    Lock lock = DistributedReadWriteLock.recoverLock(qlock, lockData);
 +    if (lock == null) {
 +      DistributedReadWriteLock locker = new DistributedReadWriteLock(qlock, lockData);
 +      if (writeLock)
 +        lock = locker.writeLock();
 +      else
 +        lock = locker.readLock();
 +    }
 +    return lock;
 +  }
 +
 +  public static Lock getReadLock(String tableId, long tid) throws Exception {
 +    return Utils.getLock(tableId, tid, false);
 +  }
 +
 +  static void checkNamespaceDoesNotExist(Instance instance, String namespace, String namespaceId, TableOperation operation)
 +      throws ThriftTableOperationException {
 +
 +    String n = Namespaces.getNameToIdMap(instance).get(namespace);
 +
 +    if (n != null && !n.equals(namespaceId))
 +      throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.NAMESPACE_EXISTS, null);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
index 20b7328,0000000..a9058b7
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
@@@ -1,92 -1,0 +1,94 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.master.tserverOps;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.Repo;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.master.Master;
 +import org.apache.accumulo.master.EventCoordinator.Listener;
 +import org.apache.accumulo.master.tableOps.MasterRepo;
 +import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.transport.TTransportException;
 +
 +public class ShutdownTServer extends MasterRepo {
 +  
 +  private static final long serialVersionUID = 1L;
 +  private static final Logger log = Logger.getLogger(ShutdownTServer.class);
 +  private TServerInstance server;
 +  private boolean force;
 +  
 +  public ShutdownTServer(TServerInstance server, boolean force) {
 +    this.server = server;
 +    this.force = force;
 +  }
 +  
 +  @Override
 +  public long isReady(long tid, Master environment) throws Exception {
 +    return 0;
 +  }
 +  
 +  @Override
 +  public Repo<Master> call(long tid, Master master) throws Exception {
 +    // suppress assignment of tablets to the server
 +    if (force) {
 +      String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZTSERVERS + "/" + server.getLocation();
 +      ZooLock.deleteLock(path);
 +      path = ZooUtil.getRoot(master.getInstance()) + Constants.ZDEADTSERVERS + "/" + server.getLocation();
 +      IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-       zoo.putPersistentData(path, "forced down".getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++      zoo.putPersistentData(path, "forced down".getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
 +      return null;
 +    }
 +    
 +    // TODO move this to isReady() and drop while loop? - ACCUMULO-1259
 +    Listener listener = master.getEventCoordinator().getListener();
 +    master.shutdownTServer(server);
 +    while (master.onlineTabletServers().contains(server)) {
 +      TServerConnection connection = master.getConnection(server);
 +      if (connection != null) {
 +        try {
 +          TabletServerStatus status = connection.getTableMap(false);
 +          if (status.tableMap != null && status.tableMap.isEmpty()) {
 +            log.info("tablet server hosts no tablets " + server);
 +            connection.halt(master.getMasterLock());
 +            log.info("tablet server asked to halt " + server);
 +            break;
 +          }
 +        } catch (TTransportException ex) {
 +          // expected
 +        } catch (Exception ex) {
 +          log.error("Error talking to tablet server " + server + ": " + ex);
 +        }
 +      }
 +      listener.waitForEvents(1000);
 +    }
 +    
 +    return null;
 +  }
 +  
 +  @Override
 +  public void undo(long tid, Master m) throws Exception {}
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index c4ab6e6,0000000..75d1e9e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -1,817 -1,0 +1,819 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.net.InetAddress;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.TreeMap;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.MasterClient;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.gc.thrift.GCMonitorService;
 +import org.apache.accumulo.core.gc.thrift.GCStatus;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ServerServices.Service;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 +import org.apache.accumulo.monitor.servlets.DefaultServlet;
 +import org.apache.accumulo.monitor.servlets.GcStatusServlet;
 +import org.apache.accumulo.monitor.servlets.JSONServlet;
 +import org.apache.accumulo.monitor.servlets.LogServlet;
 +import org.apache.accumulo.monitor.servlets.MasterServlet;
 +import org.apache.accumulo.monitor.servlets.OperationServlet;
 +import org.apache.accumulo.monitor.servlets.ProblemServlet;
 +import org.apache.accumulo.monitor.servlets.ScanServlet;
 +import org.apache.accumulo.monitor.servlets.ShellServlet;
 +import org.apache.accumulo.monitor.servlets.TServersServlet;
 +import org.apache.accumulo.monitor.servlets.TablesServlet;
 +import org.apache.accumulo.monitor.servlets.VisServlet;
 +import org.apache.accumulo.monitor.servlets.XMLServlet;
 +import org.apache.accumulo.monitor.servlets.trace.ListType;
 +import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
 +import org.apache.accumulo.monitor.servlets.trace.Summary;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.monitor.LogService;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.problems.ProblemType;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.TableInfoUtil;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.log4j.Logger;
 +import org.apache.zookeeper.KeeperException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * Serve master statistics with an embedded web server.
 + */
 +public class Monitor {
 +  private static final Logger log = Logger.getLogger(Monitor.class);
 +
 +  private static final int REFRESH_TIME = 5;
 +  private static long lastRecalc = 0L;
 +  private static double totalIngestRate = 0.0;
 +  private static double totalIngestByteRate = 0.0;
 +  private static double totalQueryRate = 0.0;
 +  private static double totalScanRate = 0.0;
 +  private static double totalQueryByteRate = 0.0;
 +  private static long totalEntries = 0L;
 +  private static int totalTabletCount = 0;
 +  private static int onlineTabletCount = 0;
 +  private static long totalHoldTime = 0;
 +  private static long totalLookups = 0;
 +  private static int totalTables = 0;
 +
 +  private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
 +    private static final long serialVersionUID = 1L;
 +
 +    private long maxDelta;
 +
 +    public MaxList(long maxDelta) {
 +      this.maxDelta = maxDelta;
 +    }
 +
 +    @Override
 +    public boolean add(Pair<Long,T> obj) {
 +      boolean result = super.add(obj);
 +
 +      if (obj.getFirst() - get(0).getFirst() > maxDelta)
 +        remove(0);
 +
 +      return result;
 +    }
 +
 +  }
 +
 +  private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
 +  private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static EventCounter lookupRateTracker = new EventCounter();
 +  private static EventCounter indexCacheHitTracker = new EventCounter();
 +  private static EventCounter indexCacheRequestTracker = new EventCounter();
 +  private static EventCounter dataCacheHitTracker = new EventCounter();
 +  private static EventCounter dataCacheRequestTracker = new EventCounter();
 +
 +  private static volatile boolean fetching = false;
 +  private static MasterMonitorInfo mmi;
 +  private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
 +  private static Exception problemException;
 +  private static GCStatus gcStatus;
 +
 +  private static Instance instance;
 +
 +  private static ServerConfiguration config;
 +
 +  private static EmbeddedWebServer server;
 +
 +  private ZooLock monitorLock;
 +
 +  private static class EventCounter {
 +
 +    Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
 +    Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
 +    Set<String> serversUpdated = new HashSet<String>();
 +
 +    void startingUpdates() {
 +      serversUpdated.clear();
 +    }
 +
 +    void updateTabletServer(String name, long sampleTime, long numEvents) {
 +      Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
 +      Pair<Long,Long> lastSample = samples.get(name);
 +
 +      if (lastSample == null || !lastSample.equals(newSample)) {
 +        samples.put(name, newSample);
 +        if (lastSample != null) {
 +          prevSamples.put(name, lastSample);
 +        }
 +      }
 +      serversUpdated.add(name);
 +    }
 +
 +    void finishedUpdating() {
 +      // remove any tablet servers not updated
 +      samples.keySet().retainAll(serversUpdated);
 +      prevSamples.keySet().retainAll(serversUpdated);
 +    }
 +
 +    double calculateRate() {
 +      double totalRate = 0;
 +
 +      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
 +        Pair<Long,Long> prevSample = entry.getValue();
 +        Pair<Long,Long> sample = samples.get(entry.getKey());
 +
 +        totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
 +      }
 +
 +      return totalRate;
 +    }
 +
 +    long calculateCount() {
 +      long count = 0;
 +
 +      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
 +        Pair<Long,Long> prevSample = entry.getValue();
 +        Pair<Long,Long> sample = samples.get(entry.getKey());
 +
 +        count += sample.getSecond() - prevSample.getSecond();
 +      }
 +
 +      return count;
 +    }
 +  }
 +
 +  public static void fetchData() {
 +    double totalIngestRate = 0.;
 +    double totalIngestByteRate = 0.;
 +    double totalQueryRate = 0.;
 +    double totalQueryByteRate = 0.;
 +    double totalScanRate = 0.;
 +    long totalEntries = 0;
 +    int totalTabletCount = 0;
 +    int onlineTabletCount = 0;
 +    long totalHoldTime = 0;
 +    long totalLookups = 0;
 +    boolean retry = true;
 +
 +    // only recalc every so often
 +    long currentTime = System.currentTimeMillis();
 +    if (currentTime - lastRecalc < REFRESH_TIME * 1000)
 +      return;
 +
 +    synchronized (Monitor.class) {
 +      if (fetching)
 +        return;
 +      fetching = true;
 +    }
 +
 +    try {
 +      while (retry) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnection(HdfsZooInstance.getInstance());
 +          if (client != null) {
 +            mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
 +            retry = false;
 +          } else {
 +            mmi = null;
 +          }
 +          Monitor.gcStatus = fetchGcStatus();
 +        } catch (Exception e) {
 +          mmi = null;
 +          log.info("Error fetching stats: " + e);
 +        } finally {
 +          if (client != null) {
 +            MasterClient.close(client);
 +          }
 +        }
 +        if (mmi == null)
 +          UtilWaitThread.sleep(1000);
 +      }
 +      if (mmi != null) {
 +        int majorCompactions = 0;
 +        int minorCompactions = 0;
 +
 +        lookupRateTracker.startingUpdates();
 +        indexCacheHitTracker.startingUpdates();
 +        indexCacheRequestTracker.startingUpdates();
 +        dataCacheHitTracker.startingUpdates();
 +        dataCacheRequestTracker.startingUpdates();
 +
 +        for (TabletServerStatus server : mmi.tServerInfo) {
 +          TableInfo summary = TableInfoUtil.summarizeTableStats(server);
 +          totalIngestRate += summary.ingestRate;
 +          totalIngestByteRate += summary.ingestByteRate;
 +          totalQueryRate += summary.queryRate;
 +          totalScanRate += summary.scanRate;
 +          totalQueryByteRate += summary.queryByteRate;
 +          totalEntries += summary.recs;
 +          totalHoldTime += server.holdTime;
 +          totalLookups += server.lookups;
 +          majorCompactions += summary.majors.running;
 +          minorCompactions += summary.minors.running;
 +          lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
 +          indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
 +          indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
 +          dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
 +          dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
 +        }
 +
 +        lookupRateTracker.finishedUpdating();
 +        indexCacheHitTracker.finishedUpdating();
 +        indexCacheRequestTracker.finishedUpdating();
 +        dataCacheHitTracker.finishedUpdating();
 +        dataCacheRequestTracker.finishedUpdating();
 +
 +        int totalTables = 0;
 +        for (TableInfo tInfo : mmi.tableMap.values()) {
 +          totalTabletCount += tInfo.tablets;
 +          onlineTabletCount += tInfo.onlineTablets;
 +          totalTables++;
 +        }
 +        Monitor.totalIngestRate = totalIngestRate;
 +        Monitor.totalTables = totalTables;
 +        totalIngestByteRate = totalIngestByteRate / 1000000.0;
 +        Monitor.totalIngestByteRate = totalIngestByteRate;
 +        Monitor.totalQueryRate = totalQueryRate;
 +        Monitor.totalScanRate = totalScanRate;
 +        totalQueryByteRate = totalQueryByteRate / 1000000.0;
 +        Monitor.totalQueryByteRate = totalQueryByteRate;
 +        Monitor.totalEntries = totalEntries;
 +        Monitor.totalTabletCount = totalTabletCount;
 +        Monitor.onlineTabletCount = onlineTabletCount;
 +        Monitor.totalHoldTime = totalHoldTime;
 +        Monitor.totalLookups = totalLookups;
 +
 +        ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
 +        ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
 +
 +        double totalLoad = 0.;
 +        for (TabletServerStatus status : mmi.tServerInfo) {
 +          if (status != null)
 +            totalLoad += status.osLoad;
 +        }
 +        loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
 +
 +        minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
 +        majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
 +
 +        lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
 +
 +        queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
 +        queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
 +
 +        scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
 +
 +        calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
 +        calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
 +      }
 +      try {
 +        Monitor.problemSummary = ProblemReports.getInstance().summarize();
 +        Monitor.problemException = null;
 +      } catch (Exception e) {
 +        log.info("Failed to obtain problem reports ", e);
 +        Monitor.problemSummary = Collections.emptyMap();
 +        Monitor.problemException = e;
 +      }
 +
 +    } finally {
 +      synchronized (Monitor.class) {
 +        fetching = false;
 +        lastRecalc = currentTime;
 +      }
 +    }
 +  }
 +
 +  private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
 +    long req = cacheReq.calculateCount();
 +    if (req > 0)
 +      hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
 +    else
 +      hitRate.add(new Pair<Long,Double>(currentTime, null));
 +  }
 +
 +  private static GCStatus fetchGcStatus() {
 +    GCStatus result = null;
 +    HostAndPort address = null;
 +    try {
 +      // Read the gc location from its lock
 +      ZooReaderWriter zk = ZooReaderWriter.getInstance();
 +      String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
 +      List<String> locks = zk.getChildren(path, null);
 +      if (locks != null && locks.size() > 0) {
 +        Collections.sort(locks);
-         address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), Constants.UTF8)).getAddress(Service.GC_CLIENT);
++        address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), UTF_8)).getAddress(Service.GC_CLIENT);
 +        GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
 +        try {
 +          result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
 +        } finally {
 +          ThriftUtil.returnClient(client);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      log.warn("Unable to contact the garbage collector at " + address, ex);
 +    }
 +    return result;
 +  }
 +
 +  public static void main(String[] args) throws Exception {
 +    SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
 +
 +    ServerOpts opts = new ServerOpts();
 +    final String app = "monitor";
 +    opts.parseArgs(app, args);
 +    String hostname = opts.getAddress();
 +
 +    Accumulo.setupLogging(app);
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    instance = HdfsZooInstance.getInstance();
 +    config = new ServerConfiguration(instance);
 +    Accumulo.init(fs, config, app);
 +    Monitor monitor = new Monitor();
 +    Accumulo.enableTracing(hostname, app);
 +    monitor.run(hostname);
 +  }
 +
 +  private static long START_TIME;
 +
 +  public void run(String hostname) {
 +    try {
 +      getMonitorLock();
 +    } catch (Exception e) {
 +      log.error("Failed to get Monitor ZooKeeper lock");
 +      throw new RuntimeException(e);
 +    }
 +
 +    Monitor.START_TIME = System.currentTimeMillis();
 +    int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
 +    try {
 +      log.debug("Creating monitor on port " + port);
 +      server = new EmbeddedWebServer(hostname, port);
 +    } catch (Throwable ex) {
 +      log.error("Unable to start embedded web server", ex);
 +      throw new RuntimeException(ex);
 +    }
 +
 +    server.addServlet(DefaultServlet.class, "/");
 +    server.addServlet(OperationServlet.class, "/op");
 +    server.addServlet(MasterServlet.class, "/master");
 +    server.addServlet(TablesServlet.class, "/tables");
 +    server.addServlet(TServersServlet.class, "/tservers");
 +    server.addServlet(ProblemServlet.class, "/problems");
 +    server.addServlet(GcStatusServlet.class, "/gc");
 +    server.addServlet(LogServlet.class, "/log");
 +    server.addServlet(XMLServlet.class, "/xml");
 +    server.addServlet(JSONServlet.class, "/json");
 +    server.addServlet(VisServlet.class, "/vis");
 +    server.addServlet(ScanServlet.class, "/scans");
 +    server.addServlet(Summary.class, "/trace/summary");
 +    server.addServlet(ListType.class, "/trace/listType");
 +    server.addServlet(ShowTrace.class, "/trace/show");
 +    if (server.isUsingSsl())
 +      server.addServlet(ShellServlet.class, "/shell");
 +    server.start();
 +
 +    try {
 +      hostname = InetAddress.getLocalHost().getHostName();
 +
 +      log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
 +
 +      String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
 +
-       ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(Constants.UTF8),
++      ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(UTF_8),
 +          NodeExistsPolicy.OVERWRITE);
 +      log.info("Set monitor address in zookeeper to " + monitorAddress);
 +    } catch (Exception ex) {
 +      log.error("Unable to set monitor HTTP address in zookeeper", ex);
 +    }
 +
 +    if (null != hostname) {
 +      LogService.startLogListener(Monitor.getSystemConfiguration(), instance.getInstanceID(), hostname);
 +    } else {
 +      log.warn("Not starting log4j listener as we could not determine address to use");
 +    }
 +
 +    new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
 +
 +    // need to regularly fetch data so plot data is updated
 +    new Daemon(new LoggingRunnable(log, new Runnable() {
 +
 +      @Override
 +      public void run() {
 +        while (true) {
 +          try {
 +            Monitor.fetchData();
 +          } catch (Exception e) {
 +            log.warn(e.getMessage(), e);
 +          }
 +
 +          UtilWaitThread.sleep(333);
 +        }
 +
 +      }
 +    }), "Data fetcher").start();
 +
 +    new Daemon(new LoggingRunnable(log, new Runnable() {
 +      @Override
 +      public void run() {
 +        while (true) {
 +          try {
 +            Monitor.fetchScans();
 +          } catch (Exception e) {
 +            log.warn(e.getMessage(), e);
 +          }
 +          UtilWaitThread.sleep(5000);
 +        }
 +      }
 +    }), "Scan scanner").start();
 +  }
 +
 +  public static class ScanStats {
 +    public final List<ActiveScan> scans;
 +    public final long fetched;
 +    ScanStats(List<ActiveScan> active) {
 +      this.scans = active;
 +      this.fetched = System.currentTimeMillis();
 +    }
 +  }
 +  static final Map<String, ScanStats> allScans = new HashMap<String, ScanStats>();
 +  public static Map<String, ScanStats> getScans() {
 +    synchronized (allScans) {
 +      return new TreeMap<String, ScanStats>(allScans);
 +    }
 +  }
 +
 +  protected static void fetchScans() throws Exception {
 +    if (instance == null)
 +      return;
 +    Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
 +    for (String server : c.instanceOperations().getTabletServers()) {
 +      Client tserver = ThriftUtil.getTServerClient(server, Monitor.getSystemConfiguration());
 +      try {
 +        List<ActiveScan> scans = tserver.getActiveScans(null, SystemCredentials.get().toThrift(instance));
 +        synchronized (allScans) {
 +          allScans.put(server, new ScanStats(scans));
 +        }
 +      } catch (Exception ex) {
 +        log.debug(ex, ex);
 +      } finally {
 +        ThriftUtil.returnClient(tserver);
 +      }
 +    }
 +    // Age off old scan information
 +    Iterator<Entry<String,ScanStats>> entryIter = allScans.entrySet().iterator();
 +    long now = System.currentTimeMillis();
 +    while (entryIter.hasNext()) {
 +      Entry<String,ScanStats> entry = entryIter.next();
 +      if (now - entry.getValue().fetched > 5 * 60 * 1000) {
 +        entryIter.remove();
 +      }
 +    }
 +  }
 +
 +  /**
 +   * Get the monitor lock in ZooKeeper
 +   */
 +  private void getMonitorLock() throws KeeperException, InterruptedException {
 +    final String zRoot = ZooUtil.getRoot(instance);
 +    final String monitorPath = zRoot + Constants.ZMONITOR;
 +    final String monitorLockPath = zRoot + Constants.ZMONITOR_LOCK;
 +
 +    // Ensure that everything is kosher with ZK as this has changed.
 +    ZooReaderWriter zoo = ZooReaderWriter.getInstance();
 +    if (zoo.exists(monitorPath)) {
 +      byte[] data = zoo.getData(monitorPath, null);
 +      // If the node isn't empty, it's from a previous install (has hostname:port for HTTP server)
 +      if (0 != data.length) {
 +        // Recursively delete from that parent node
 +        zoo.recursiveDelete(monitorPath, NodeMissingPolicy.SKIP);
 +
 +        // And then make the nodes that we expect for the incoming ephemeral nodes
 +        zoo.putPersistentData(monitorPath, new byte[0], NodeExistsPolicy.FAIL);
 +        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
 +      } else if (!zoo.exists(monitorLockPath)) {
 +        // monitor node in ZK exists and is empty as we expect
 +        // but the monitor/lock node does not
 +        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
 +      }
 +    } else {
 +      // 1.5.0 and earlier
 +      zoo.putPersistentData(zRoot + Constants.ZMONITOR, new byte[0], NodeExistsPolicy.FAIL);
 +      if (!zoo.exists(monitorLockPath)) {
 +        // Somehow the monitor node exists but not monitor/lock
 +        zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
 +      }
 +    }
 +
 +    // Get a ZooLock for the monitor
 +    while (true) {
 +      MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher();
 +      monitorLock = new ZooLock(monitorLockPath);
 +      monitorLock.lockAsync(monitorLockWatcher, new byte[0]);
 +
 +      monitorLockWatcher.waitForChange();
 +
 +      if (monitorLockWatcher.acquiredLock) {
 +        break;
 +      }
 +
 +      if (!monitorLockWatcher.failedToAcquireLock) {
 +        throw new IllegalStateException("monitor lock in unknown state");
 +      }
 +
 +      monitorLock.tryToCancelAsyncLockOrUnlock();
 +
 +      UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));
 +    }
 +
 +    log.info("Got Monitor lock.");
 +  }
 +
 +  /**
 +   * Async Watcher for monitor lock
 +   */
 +  private static class MoniterLockWatcher implements ZooLock.AsyncLockWatcher {
 +
 +    boolean acquiredLock = false;
 +    boolean failedToAcquireLock = false;
 +
 +    @Override
 +    public void lostLock(LockLossReason reason) {
 +      Halt.halt("Monitor lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
 +    }
 +
 +    @Override
 +    public void unableToMonitorLockNode(final Throwable e) {
 +      Halt.halt(-1, new Runnable() {
 +        @Override
 +        public void run() {
 +          log.fatal("No longer able to monitor Monitor lock node", e);
 +        }
 +      });
 +
 +    }
 +
 +    @Override
 +    public synchronized void acquiredLock() {
 +      if (acquiredLock || failedToAcquireLock) {
 +        Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      acquiredLock = true;
 +      notifyAll();
 +    }
 +
 +    @Override
 +    public synchronized void failedToAcquireLock(Exception e) {
 +      log.warn("Failed to get monitor lock " + e);
 +
 +      if (acquiredLock) {
 +        Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
 +      }
 +
 +      failedToAcquireLock = true;
 +      notifyAll();
 +    }
 +
 +    public synchronized void waitForChange() {
 +      while (!acquiredLock && !failedToAcquireLock) {
 +        try {
 +          wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +  }
 +
 +  public static MasterMonitorInfo getMmi() {
 +    return mmi;
 +  }
 +
 +  public static int getTotalTables() {
 +    return totalTables;
 +  }
 +
 +  public static int getTotalTabletCount() {
 +    return totalTabletCount;
 +  }
 +
 +  public static int getOnlineTabletCount() {
 +    return onlineTabletCount;
 +  }
 +
 +  public static long getTotalEntries() {
 +    return totalEntries;
 +  }
 +
 +  public static double getTotalIngestRate() {
 +    return totalIngestRate;
 +  }
 +
 +  public static double getTotalIngestByteRate() {
 +    return totalIngestByteRate;
 +  }
 +
 +  public static double getTotalQueryRate() {
 +    return totalQueryRate;
 +  }
 +
 +  public static double getTotalScanRate() {
 +    return totalScanRate;
 +  }
 +
 +  public static double getTotalQueryByteRate() {
 +    return totalQueryByteRate;
 +  }
 +
 +  public static long getTotalHoldTime() {
 +    return totalHoldTime;
 +  }
 +
 +  public static Exception getProblemException() {
 +    return problemException;
 +  }
 +
 +  public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
 +    return problemSummary;
 +  }
 +
 +  public static GCStatus getGcStatus() {
 +    return gcStatus;
 +  }
 +
 +  public static long getTotalLookups() {
 +    return totalLookups;
 +  }
 +
 +  public static long getStartTime() {
 +    return START_TIME;
 +  }
 +
 +  public static List<Pair<Long,Double>> getLoadOverTime() {
 +    synchronized (loadOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(loadOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getIngestRateOverTime() {
 +    synchronized (ingestRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
 +    synchronized (ingestByteRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
 +    synchronized (recoveriesOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
 +    synchronized (minorCompactionsOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
 +    synchronized (majorCompactionsOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getLookupsOverTime() {
 +    synchronized (lookupsOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
 +    }
 +  }
 +
 +  public static double getLookupRate() {
 +    return lookupRateTracker.calculateRate();
 +  }
 +
 +  public static List<Pair<Long,Integer>> getQueryRateOverTime() {
 +    synchronized (queryRateOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Integer>> getScanRateOverTime() {
 +    synchronized (scanRateOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
 +    synchronized (queryByteRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
 +    synchronized (indexCacheHitRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
 +    }
 +  }
 +
 +  public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
 +    synchronized (dataCacheHitRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
 +    }
 +  }
 +
 +  public static AccumuloConfiguration getSystemConfiguration() {
 +    return config.getConfiguration();
 +  }
 +
 +  public static Instance getInstance() {
 +    return instance;
 +  }
 +
 +  public static boolean isUsingSsl() {
 +    return server.isUsingSsl();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index 5e3e475,0000000..78dd95e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -1,140 -1,0 +1,141 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.util.Collection;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.util.TTimeoutTransport;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class ZooKeeperStatus implements Runnable {
 +  
 +  private static final Logger log = Logger.getLogger(ZooKeeperStatus.class);
 +  
 +  private volatile boolean stop = false;
 +  
 +  public static class ZooKeeperState implements Comparable<ZooKeeperState> {
 +    public final String keeper;
 +    public final String mode;
 +    public final int clients;
 +    
 +    public ZooKeeperState(String keeper, String mode, int clients) {
 +      this.keeper = keeper;
 +      this.mode = mode;
 +      this.clients = clients;
 +    }
 +    
 +    @Override
 +    public int compareTo(ZooKeeperState other) {
 +      if (this == other) {
 +        return 0;
 +      } else if (other == null) {
 +        return 1;
 +      } else {
 +        if (this.keeper == other.keeper) {
 +          return 0;
 +        } else if (null == this.keeper) {
 +          return -1;
 +        } else if (null == other.keeper) {
 +          return 1;
 +        } else {
 +          return this.keeper.compareTo(other.keeper);
 +        }
 +      }
 +    }
 +  }
 +  
 +  private static SortedSet<ZooKeeperState> status = new TreeSet<ZooKeeperState>();
 +  
 +  public static Collection<ZooKeeperState> getZooKeeperStatus() {
 +    return status;
 +  }
 +  
 +  public void stop() {
 +    this.stop = true;
 +  }
 +  
 +  @Override
 +  public void run() {
 +    
 +    while (!stop) {
 +      
 +      TreeSet<ZooKeeperState> update = new TreeSet<ZooKeeperState>();
 +      
 +      String zookeepers[] = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST).split(",");
 +      for (String keeper : zookeepers) {
 +        int clients = 0;
 +        String mode = "unknown";
 +        
 +        String[] parts = keeper.split(":");
 +        TTransport transport = null;
 +        try {
 +          HostAndPort addr;
 +          if (parts.length > 1)
 +            addr = HostAndPort.fromParts(parts[0], Integer.parseInt(parts[1]));
 +          else
 +            addr = HostAndPort.fromParts(parts[0], 2181);
 +          
 +          transport = TTimeoutTransport.create(addr, 10 * 1000l);
-           transport.write("stat\n".getBytes(Constants.UTF8), 0, 5);
++          transport.write("stat\n".getBytes(UTF_8), 0, 5);
 +          StringBuilder response = new StringBuilder();
 +          try {
 +            transport.flush();
 +            byte[] buffer = new byte[1024 * 100];
 +            int n = 0;
 +            while ((n = transport.read(buffer, 0, buffer.length)) > 0) {
-               response.append(new String(buffer, 0, n, Constants.UTF8));
++              response.append(new String(buffer, 0, n, UTF_8));
 +            }
 +          } catch (TTransportException ex) {
 +            // happens at EOF
 +          }
 +          for (String line : response.toString().split("\n")) {
 +            if (line.startsWith(" "))
 +              clients++;
 +            if (line.startsWith("Mode"))
 +              mode = line.split(":")[1];
 +          }
 +          update.add(new ZooKeeperState(keeper, mode, clients));
 +        } catch (Exception ex) {
 +          log.info("Exception talking to zookeeper " + keeper, ex);
 +          update.add(new ZooKeeperState(keeper, "Down", -1));
 +        } finally {
 +          if (transport != null) {
 +            try {
 +              transport.close();
 +            } catch (Exception ex) {
 +              log.error(ex, ex);
 +            }
 +          }
 +        }
 +      }
 +      status = update;
 +      UtilWaitThread.sleep(5 * 1000);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index d896040,0000000..7812603
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@@ -1,283 -1,0 +1,285 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.PrintWriter;
 +import java.io.StringWriter;
 +import java.io.UnsupportedEncodingException;
 +import java.net.URLDecoder;
 +import java.net.URLEncoder;
 +import java.util.Date;
 +import java.util.List;
 +import java.util.TimerTask;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.http.Cookie;
 +import javax.servlet.http.HttpServlet;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.monitor.DedupedLogEvent;
 +import org.apache.accumulo.server.monitor.LogService;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.Logger;
 +
 +abstract public class BasicServlet extends HttpServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +  protected static final Logger log = Logger.getLogger(BasicServlet.class);
 +  static String cachedInstanceName = null;
 +  private static String bannerText;
 +  private static String bannerColor;
 +  private static String bannerBackground;
 +
 +  abstract protected String getTitle(HttpServletRequest req);
 +
 +  @Override
 +  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    StringBuilder sb = new StringBuilder();
 +    try {
 +      Monitor.fetchData();
 +      bannerText = sanitize(Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_TEXT));
 +      bannerColor = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_COLOR).replace("'", "&#39;");
 +      bannerBackground = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_BACKGROUND).replace("'", "&#39;");
 +      pageStart(req, resp, sb);
 +      pageBody(req, resp, sb);
 +      pageEnd(req, resp, sb);
 +    } catch (Throwable t) {
 +      log.error("Error building page " + req.getRequestURI(), t);
 +      sb.append("\n<pre>\n");
 +      StringWriter sw = new StringWriter();
 +      t.printStackTrace(new PrintWriter(sw));
 +      sb.append(sanitize(sw.getBuffer().toString()));
 +      sb.append("</pre>\n");
 +    } finally {
 +      resp.getWriter().print(sb);
 +      resp.getWriter().flush();
 +    }
 +  }
 +
 +  @Override
 +  protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    doGet(req, resp);
 +  }
 +
 +  private static final String DEFAULT_CONTENT_TYPE = "text/html";
 +
 +  public static final void setCookie(HttpServletResponse resp, String name, String value) {
 +    resp.addCookie(new Cookie(name, value));
 +  }
 +
 +  public static final String getCookieValue(HttpServletRequest req, String name) {
 +    if (req.getCookies() != null)
 +      for (Cookie c : req.getCookies())
 +        if (c.getName().equals(name))
 +          return c.getValue();
 +    return null;
 +  }
 +
 +  protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
 +    resp.setContentType(DEFAULT_CONTENT_TYPE);
 +    int refresh = -1;
 +    String refreshStr = getCookieValue(req, "page.refresh.rate");
 +    if (refreshStr != null) {
 +      try {
 +        refresh = Integer.parseInt(BasicServlet.decode(refreshStr));
 +      } catch (NumberFormatException e) {
 +        // ignore improperly formatted user cookie
 +      }
 +    }
 +    synchronized (BasicServlet.class) {
 +      // Learn our instance name asynchronously so we don't hang up if zookeeper is down
 +      if (cachedInstanceName == null) {
 +        SimpleTimer.getInstance().schedule(new TimerTask() {
 +          @Override
 +          public void run() {
 +            synchronized (BasicServlet.class) {
 +              if (cachedInstanceName == null) {
 +                cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName();
 +              }
 +            }
 +          }
 +        }, 1000);
 +      }
 +    }
 +
 +    // BEGIN PAGE
 +    sb.append("<!--\n");
 +    sb.append("  Licensed to the Apache Software Foundation (ASF) under one or more\n");
 +    sb.append("  contributor license agreements.  See the NOTICE file distributed with\n");
 +    sb.append("  this work for additional information regarding copyright ownership.\n");
 +    sb.append("  The ASF licenses this file to You under the Apache License, Version 2.0\n");
 +    sb.append("  (the \"License\"); you may not use this file except in compliance with\n");
 +    sb.append("  the License.  You may obtain a copy of the License at\n");
 +    sb.append("\n");
 +    sb.append("    http://www.apache.org/licenses/LICENSE-2.0\n");
 +    sb.append("\n");
 +    sb.append("  Unless required by applicable law or agreed to in writing, software\n");
 +    sb.append("  distributed under the License is distributed on an \"AS IS\" BASIS,\n");
 +    sb.append("  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n");
 +    sb.append("  See the License for the specific language governing permissions and\n");
 +    sb.append("  limitations under the License.\n");
 +    sb.append("-->\n");
 +    sb.append("<html>\n");
 +
 +    // BEGIN HEADER
 +    sb.append("<head>\n");
 +    sb.append("<title>").append(getTitle(req)).append(" - Accumulo ").append(Constants.VERSION).append("</title>\n");
 +    if ((refresh > 0) && (req.getRequestURI().startsWith("/vis") == false) && (req.getRequestURI().startsWith("/shell") == false))
 +      sb.append("<meta http-equiv='refresh' content='" + refresh + "' />\n");
 +    sb.append("<meta http-equiv='Content-Type' content='").append(DEFAULT_CONTENT_TYPE).append("' />\n");
 +    sb.append("<meta http-equiv='Content-Script-Type' content='text/javascript' />\n");
 +    sb.append("<meta http-equiv='Content-Style-Type' content='text/css' />\n");
 +    sb.append("<link rel='shortcut icon' type='image/jpg' href='/web/favicon.png' />\n");
 +    sb.append("<link rel='stylesheet' type='text/css' href='/web/screen.css' media='screen' />\n");
 +    sb.append("<script src='/web/functions.js' type='text/javascript'></script>\n");
 +
 +    sb.append("<!--[if lte IE 8]><script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/excanvas.min.js\"></script><![endif]-->\n");
 +    sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.js\"></script>\n");
 +    sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.flot.js\"></script>\n");
 +
 +    sb.append("</head>\n");
 +
 +    // BEGIN BODY OPENING
 +    sb.append("\n<body ").append(getBodyAttributes()).append(">\n");
 +    sb.append("<div id='content-wrapper'>\n");
 +    sb.append("<div id='content'>\n");
 +    sb.append("<div id='header'>");
 +    if (!bannerText.isEmpty()) {
 +      sb.append("<div id='banner' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
 +          .append("</div>\n");
 +    }
 +    sb.append("<div id='headertitle'>");
 +    sb.append("<h1>").append(getTitle(req)).append("</h1></div>\n");
 +    sb.append("<div id='subheader'>Instance&nbsp;Name:&nbsp;").append(cachedInstanceName).append("&nbsp;&nbsp;&nbsp;Version:&nbsp;").append(Constants.VERSION)
 +        .append("\n");
 +    sb.append("<br><span class='smalltext'>Instance&nbsp;ID:&nbsp;").append(HdfsZooInstance.getInstance().getInstanceID()).append("</span>\n");
 +    sb.append("<br><span class='smalltext'>").append(new Date().toString().replace(" ", "&nbsp;")).append("</span>");
 +    sb.append("</div>\n"); // end <div id='subheader'>
 +    sb.append("</div>\n"); // end <div id='header'>
 +
 +    // BEGIN LEFT SIDE
 +    sb.append("<div id='nav'>\n");
 +    sb.append("<span id='nav-title'><a href='/'>Overview</a></span><br />\n");
 +    sb.append("<hr />\n");
 +    sb.append("<a href='/master'>Master&nbsp;Server</a><br />\n");
 +    sb.append("<a href='/tservers'>Tablet&nbsp;Servers</a><br />\n");
 +    sb.append("<a href='/scans'>Active&nbsp;Scans</a><br />\n");
 +    sb.append("<a href='/vis'>Server Activity</a><br />\n");
 +    sb.append("<a href='/gc'>Garbage&nbsp;Collector</a><br />\n");
 +    sb.append("<a href='/tables'>Tables</a><br />\n");
 +    sb.append("<a href='/trace/summary?minutes=10'>Recent&nbsp;Traces</a><br />\n");
 +    List<DedupedLogEvent> dedupedLogEvents = LogService.getInstance().getEvents();
 +    int numLogs = dedupedLogEvents.size();
 +    boolean logsHaveError = false;
 +    for (DedupedLogEvent dedupedLogEvent : dedupedLogEvents)
 +      if (dedupedLogEvent.getEvent().getLevel().isGreaterOrEqual(Level.ERROR)) {
 +        logsHaveError = true;
 +        break;
 +      }
 +    if (numLogs > 0)
 +      sb.append("<span class='" + (logsHaveError ? "error" : "warning") + "'><a href='/log'>Recent&nbsp;Logs&nbsp;<span class='smalltext'>(" + numLogs
 +          + ")</a></span></span><br />\n");
 +    int numProblems = Monitor.getProblemSummary().entrySet().size();
 +    if (numProblems > 0)
 +      sb.append("<span class='error'><a href='/problems'>Table&nbsp;Problems&nbsp;<span class='smalltext'>(" + numProblems + ")</a></span></span><br />\n");
 +    sb.append("<hr />\n");
 +    sb.append("<a href='/xml'>XML</a><br />\n");
 +    sb.append("<a href='/json'>JSON</a><hr />\n");
 +    if (Monitor.isUsingSsl())
 +      sb.append("<a href='/shell'>Shell</a><hr />\n");
 +    sb.append("<div class='smalltext'>[<a href='").append("/op?action=refresh&value=").append(refresh < 1 ? "5" : "-1");
 +    sb.append("&redir=").append(currentPage(req)).append("'>");
 +    sb.append(refresh < 1 ? "en" : "dis").append("able&nbsp;auto-refresh</a>]</div>\n");
 +    sb.append("</div>\n"); // end <div id='nav'>
 +
 +    sb.append("<div id='main'");
 +    if (bannerText.isEmpty())
 +      sb.append(" style='bottom:0'");
 +    sb.append(">\n");
 +    sb.append("<!-- BEGIN MAIN BODY CONTENT -->\n\n");
 +  }
 +
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
 +    sb.append("This page intentionally left blank.");
 +  }
 +
 +  protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
 +    sb.append("\n<!-- END MAIN BODY CONTENT -->\n");
 +    sb.append("</div>\n"); // end <div id='main'>
 +
 +    // BEGIN FOOTER
 +    sb.append("</div>\n"); // end <div id='content'>
 +    sb.append("</div>\n"); // end <div id='content-wrapper'>
 +    if (!bannerText.isEmpty()) {
 +      sb.append("<div id='footer' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
 +          .append("</div>\n");
 +    }
 +    sb.append("</body>\n");
 +    sb.append("</html>\n");
 +  }
 +
 +  /**
 +   * Allow the concrete servlet implementation to provide attributes on the body HTML tag, such as 'onload', which can be used to call Javascript methods on
 +   * page load. By default, nothing is specified.
 +   */
 +  protected String getBodyAttributes() {
 +    return "";
 +  }
 +
 +  public static String encode(String s) {
 +    try {
-       return URLEncoder.encode(s, Constants.UTF8.name());
++      return URLEncoder.encode(s, UTF_8.name());
 +    } catch (UnsupportedEncodingException e) {
-       Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++      Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
 +      throw new AssertionError(e); // can't happen with UTF-8
 +    }
 +  }
 +
 +  public static String decode(String s) {
 +    try {
-       return URLDecoder.decode(s, Constants.UTF8.name());
++      return URLDecoder.decode(s, UTF_8.name());
 +    } catch (UnsupportedEncodingException e) {
-       Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++      Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
 +      throw new AssertionError(e); // can't happen with UTF-8
 +    }
 +  }
 +
 +  public static String sanitize(String xml) {
 +    return xml.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
 +  }
 +
 +  public static String currentPage(HttpServletRequest req) {
 +    String redir = req.getRequestURI();
 +    if (req.getQueryString() != null)
 +      redir += "?" + req.getQueryString();
 +    return encode(redir);
 +  }
 +
 +  protected static void banner(StringBuilder sb, String klass, String text) {
 +    sb.append("<br />\n<h2 class='").append(klass).append("'>").append(text).append("</h2>\n");
 +  }
 +
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index d88bd7c,0000000..bb7e690
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -1,346 -1,0 +1,347 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
++import static com.google.common.base.Charsets.UTF_8;
++
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Calendar;
 +import java.util.Date;
 +import java.util.List;
 +import java.util.TimeZone;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.ServletOutputStream;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
- import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.util.Duration;
 +import org.apache.accumulo.core.util.NumUtil;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.volume.VolumeConfiguration;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.ZooKeeperStatus;
 +import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
 +import org.apache.accumulo.monitor.util.celltypes.NumberType;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.hadoop.fs.ContentSummary;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +
 +public class DefaultServlet extends BasicServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Accumulo Overview";
 +  }
 +
 +  private void getResource(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 +    try {
 +      String path = req.getRequestURI();
 +
 +      if (path.endsWith(".jpg"))
 +        resp.setContentType("image/jpeg");
 +
 +      if (path.endsWith(".html"))
 +        resp.setContentType("text/html");
 +
 +      path = path.substring(1);
 +      InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path);
 +      ServletOutputStream out = resp.getOutputStream();
 +      try {
 +        if (data != null) {
 +          byte[] buffer = new byte[1024];
 +          int n;
 +          while ((n = data.read(buffer)) > 0)
 +            out.write(buffer, 0, n);
 +        } else {
-           out.write(("could not get resource " + path + "").getBytes(Constants.UTF8));
++          out.write(("could not get resource " + path + "").getBytes(UTF_8));
 +        }
 +      } finally {
 +        if (data != null)
 +          data.close();
 +      }
 +    } catch (Throwable t) {
 +      log.error(t, t);
 +      throw new IOException(t);
 +    }
 +  }
 +
 +  @Override
 +  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    if (req.getRequestURI().startsWith("/web"))
 +      getResource(req, resp);
 +    else if (req.getRequestURI().startsWith("/monitor"))
 +      resp.sendRedirect("/master");
 +    else if (req.getRequestURI().startsWith("/errors"))
 +      resp.sendRedirect("/problems");
 +    else
 +      super.doGet(req, resp);
 +  }
 +
 +  public static final int GRAPH_WIDTH = 450;
 +  public static final int GRAPH_HEIGHT = 150;
 +
 +  private static void plotData(StringBuilder sb, String title, @SuppressWarnings("rawtypes") List data, boolean points) {
 +    plotData(sb, title, points, new ArrayList<String>(), data);
 +  }
 +
 +  @SuppressWarnings("rawtypes")
 +  private static void plotData(StringBuilder sb, String title, boolean points, List<String> labels, List... series) {
 +    sb.append("<div class=\"plotHeading\">");
 +    sb.append(title);
 +    sb.append("</div>");
 +    sb.append("</br>");
 +    String id = "c" + title.hashCode();
 +    sb.append("<div id=\"" + id + "\" style=\"width:" + GRAPH_WIDTH + "px;height:" + GRAPH_HEIGHT + "px;\"></div>\n");
 +
 +    sb.append("<script type=\"text/javascript\">\n");
 +    sb.append("$(function () {\n");
 +
 +    for (int i = 0; i < series.length; i++) {
 +
 +      @SuppressWarnings("unchecked")
 +      List<Pair<Long,? extends Number>> data = series[i];
 +      sb.append("    var d" + i + " = [");
 +
 +      String sep = "";
 +      for (Pair<Long,? extends Number> point : data) {
 +        if (point.getSecond() == null)
 +          continue;
 +
 +        String y;
 +        if (point.getSecond() instanceof Double)
 +          y = String.format("%1.2f", point.getSecond());
 +        else
 +          y = point.getSecond().toString();
 +
 +        sb.append(sep);
 +        sep = ",";
 +        sb.append("[" + utc2local(point.getFirst()) + "," + y + "]");
 +      }
 +      sb.append("    ];\n");
 +    }
 +
 +    String opts = "lines: { show: true }";
 +    if (points)
 +      opts = "points: { show: true, radius: 1 }";
 +
 +    sb.append("    $.plot($(\"#" + id + "\"),");
 +    String sep = "";
 +
 +    String colors[] = new String[] {"red", "blue", "green", "black"};
 +
 +    sb.append("[");
 +    for (int i = 0; i < series.length; i++) {
 +      sb.append(sep);
 +      sep = ",";
 +      sb.append("{ ");
 +      if (labels.size() > 0) {
 +        sb.append("label: \"" + labels.get(i) + "\", ");
 +      }
 +      sb.append("data: d" + i + ", " + opts + ", color:\"" + colors[i] + "\" }");
 +    }
 +    sb.append("], ");
 +    sb.append("{yaxis:{}, xaxis:{mode:\"time\",minTickSize: [1, \"minute\"],timeformat: \"%H:%M<br />" + getShortTZName() + "\", ticks:3}});");
 +    sb.append("   });\n");
 +    sb.append("</script>\n");
 +  }
 +
 +  /**
 +   * Shows the current time zone (based on the current time) short name
 +   */
 +  private static String getShortTZName() {
 +    TimeZone tz = TimeZone.getDefault();
 +    return tz.getDisplayName(tz.inDaylightTime(new Date()), TimeZone.SHORT);
 +  }
 +
 +  /**
 +   * Converts a unix timestamp in UTC to one that is relative to the local timezone
 +   */
 +  private static Long utc2local(Long utcMillis) {
 +    Calendar currentCalendar = Calendar.getInstance(); // default timezone
 +    currentCalendar.setTimeInMillis(utcMillis + currentCalendar.getTimeZone().getOffset(utcMillis));
 +    return currentCalendar.getTime().getTime();
 +  }
 +
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws IOException {
 +
 +    sb.append("<table class='noborder'>\n");
 +    sb.append("<tr>\n");
 +
 +    sb.append("<td class='noborder'>\n");
 +    doAccumuloTable(sb);
 +    sb.append("</td>\n");
 +
 +    sb.append("<td class='noborder'>\n");
 +    doZooKeeperTable(sb);
 +    sb.append("</td>\n");
 +
 +    sb.append("</tr></table>\n");
 +    sb.append("<br/>\n");
 +
 +    sb.append("<p/><table class=\"noborder\">\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Ingest (Entries/s)", Monitor.getIngestRateOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Scan (Entries/s)", false, Arrays.asList("Read", "Returned"), Monitor.getScanRateOverTime(), Monitor.getQueryRateOverTime());
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Ingest (MB/s)", Monitor.getIngestByteRateOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Scan (MB/s)", Monitor.getQueryByteRateOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Load Average", Monitor.getLoadOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Seeks", Monitor.getLookupsOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Minor Compactions", Monitor.getMinorCompactionsOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Major Compactions", Monitor.getMajorCompactionsOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Index Cache Hit Rate", Monitor.getIndexCacheHitRateOverTime(), true);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Data Cache Hit Rate", Monitor.getDataCacheHitRateOverTime(), true);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("</table>\n");
 +  }
 +
 +  private void doAccumuloTable(StringBuilder sb) throws IOException {
 +    // Accumulo
 +    VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
 +    MasterMonitorInfo info = Monitor.getMmi();
 +    sb.append("<table>\n");
 +    sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
 +    if (info == null) {
 +      sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
 +    } else {
 +      long totalAcuBytesUsed = 0l;
 +      long totalHdfsBytesUsed = 0l;
 +
 +      try {
 +        for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
 +          final Path basePath = new Path(baseDir);
 +          final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
 +
 +          try {
 +            // Calculate the amount of space used by Accumulo on the FileSystem
 +            ContentSummary accumuloSummary = fs.getContentSummary(basePath);
 +            long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed();
 +            totalAcuBytesUsed += bytesUsedByAcuOnFs;
 +
 +            // Catch the overflow -- this is big data
 +            if (totalAcuBytesUsed < bytesUsedByAcuOnFs) {
 +              log.debug("Overflowed long in bytes used by Accumulo for " + baseDir);
 +              totalAcuBytesUsed = 0l;
 +              break;
 +            }
 +
 +            // Calculate the total amount of space used on the FileSystem
 +            ContentSummary volumeSummary = fs.getContentSummary(new Path("/"));
 +            long bytesUsedOnVolume = volumeSummary.getSpaceConsumed();
 +            totalHdfsBytesUsed += bytesUsedOnVolume;
 +
 +            // Catch the overflow -- this is big data
 +            if (totalHdfsBytesUsed < bytesUsedOnVolume) {
 +              log.debug("Overflowed long in bytes used in HDFS for " + baseDir);
 +              totalHdfsBytesUsed = 0;
 +              break;
 +            }
 +          } catch (Exception ex) {
 +            log.trace("Unable to get disk usage information for " + baseDir, ex);
 +          }
 +        }
 +
 +        String diskUsed = "Unknown";
 +        String consumed = null;
 +        if (totalAcuBytesUsed > 0) {
 +          // Convert Accumulo usage to a readable String
 +          diskUsed = bytes(totalAcuBytesUsed);
 +
 +          if (totalHdfsBytesUsed > 0) {
 +            // Compute amount of space used by Accumulo as a percentage of total space usage.
 +            consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed);
 +          }
 +        }
 +
 +        boolean highlight = false;
 +        tableRow(sb, (highlight = !highlight), "Disk&nbsp;Used", diskUsed);
 +        if (null != consumed)
 +          tableRow(sb, (highlight = !highlight), "%&nbsp;of&nbsp;Used&nbsp;DFS", consumed);
 +        tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables()));
 +        tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet&nbsp;Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));
 +        tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Dead&nbsp;Tablet&nbsp;Servers</a>", NumberType.commas(info.deadTabletServers.size(), 0, 0));
 +        tableRow(sb, (highlight = !highlight), "Tablets", NumberType.commas(Monitor.getTotalTabletCount(), 1, Long.MAX_VALUE));
 +        tableRow(sb, (highlight = !highlight), "Entries", NumberType.commas(Monitor.getTotalEntries()));
 +        tableRow(sb, (highlight = !highlight), "Lookups", NumberType.commas(Monitor.getTotalLookups()));
 +        tableRow(sb, (highlight = !highlight), "Uptime", Duration.format(System.currentTimeMillis() - Monitor.getStartTime()));
 +      } catch (Exception e) {
 +        log.debug(e, e);
 +      }
 +    }
 +    sb.append("</table>\n");
 +  }
 +
 +  private void doZooKeeperTable(StringBuilder sb) throws IOException {
 +    // Zookeepers
 +    sb.append("<table>\n");
 +    sb.append("<tr><th colspan='3'>Zookeeper</th></tr>\n");
 +    sb.append("<tr><th>Server</th><th>Mode</th><th>Clients</th></tr>\n");
 +
 +    boolean highlight = false;
 +    for (ZooKeeperState k : ZooKeeperStatus.getZooKeeperStatus()) {
 +      if (k.clients >= 0) {
 +        tableRow(sb, (highlight = !highlight), k.keeper, k.mode, k.clients);
 +      } else {
 +        tableRow(sb, false, k.keeper, "<span class='error'>Down</span>", "");
 +      }
 +    }
 +    sb.append("</table>\n");
 +  }
 +
 +  private static String bytes(long big) {
 +    return NumUtil.bigNumberForSize(big);
 +  }
 +
 +  public static void tableRow(StringBuilder sb, boolean highlight, Object... cells) {
 +    sb.append(highlight ? "<tr class='highlight'>" : "<tr>");
 +    for (int i = 0; i < cells.length; ++i) {
 +      Object cell = cells[i];
 +      String cellValue = cell == null ? "" : String.valueOf(cell).trim();
 +      sb.append("<td class='").append(i < cells.length - 1 ? "left" : "right").append("'>").append(cellValue.isEmpty() ? "-" : cellValue).append("</td>");
 +    }
 +    sb.append("</tr>\n");
 +  }
 +}


[05/50] [abbrv] Merge branch '1.5' into 1.6

Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 03fe069,0000000..94be0bb
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -1,3928 -1,0 +1,3929 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.tserver;
 +
++import static com.google.common.base.Charsets.UTF_8;
 +import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.lang.management.GarbageCollectorMXBean;
 +import java.lang.management.ManagementFactory;
 +import java.net.Socket;
 +import java.net.UnknownHostException;
 +import java.nio.ByteBuffer;
 +import java.security.SecureRandom;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.EnumMap;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +import java.util.SortedMap;
 +import java.util.SortedSet;
 +import java.util.TimerTask;
 +import java.util.TreeMap;
 +import java.util.TreeSet;
 +import java.util.concurrent.ArrayBlockingQueue;
 +import java.util.concurrent.BlockingDeque;
 +import java.util.concurrent.CancellationException;
 +import java.util.concurrent.ExecutionException;
 +import java.util.concurrent.LinkedBlockingDeque;
 +import java.util.concurrent.RunnableFuture;
 +import java.util.concurrent.ThreadPoolExecutor;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.TimeoutException;
 +import java.util.concurrent.atomic.AtomicBoolean;
 +import java.util.concurrent.atomic.AtomicInteger;
 +import java.util.concurrent.atomic.AtomicLong;
 +import java.util.concurrent.atomic.AtomicReference;
 +
 +import javax.management.ObjectName;
 +import javax.management.StandardMBean;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.CompressedIterators;
 +import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
 +import org.apache.accumulo.core.client.impl.ScannerImpl;
 +import org.apache.accumulo.core.client.impl.Tables;
 +import org.apache.accumulo.core.client.impl.TabletType;
 +import org.apache.accumulo.core.client.impl.Translator;
 +import org.apache.accumulo.core.client.impl.Translator.TKeyExtentTranslator;
 +import org.apache.accumulo.core.client.impl.Translator.TRangeTranslator;
 +import org.apache.accumulo.core.client.impl.Translators;
 +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.constraints.Constraint.Environment;
 +import org.apache.accumulo.core.constraints.Violations;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Column;
 +import org.apache.accumulo.core.data.ConstraintViolationSummary;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.thrift.InitialMultiScan;
 +import org.apache.accumulo.core.data.thrift.InitialScan;
 +import org.apache.accumulo.core.data.thrift.IterInfo;
 +import org.apache.accumulo.core.data.thrift.MapFileInfo;
 +import org.apache.accumulo.core.data.thrift.MultiScanResult;
 +import org.apache.accumulo.core.data.thrift.ScanResult;
 +import org.apache.accumulo.core.data.thrift.TCMResult;
 +import org.apache.accumulo.core.data.thrift.TCMStatus;
 +import org.apache.accumulo.core.data.thrift.TColumn;
 +import org.apache.accumulo.core.data.thrift.TCondition;
 +import org.apache.accumulo.core.data.thrift.TConditionalMutation;
 +import org.apache.accumulo.core.data.thrift.TConditionalSession;
 +import org.apache.accumulo.core.data.thrift.TKey;
 +import org.apache.accumulo.core.data.thrift.TKeyExtent;
 +import org.apache.accumulo.core.data.thrift.TKeyValue;
 +import org.apache.accumulo.core.data.thrift.TMutation;
 +import org.apache.accumulo.core.data.thrift.TRange;
 +import org.apache.accumulo.core.data.thrift.UpdateErrors;
 +import org.apache.accumulo.core.iterators.IterationInterruptedException;
 +import org.apache.accumulo.core.master.thrift.Compacting;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletLoadState;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.metadata.MetadataTable;
 +import org.apache.accumulo.core.metadata.RootTable;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.security.AuthorizationContainer;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
 +import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
 +import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
 +import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
 +import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
 +import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
 +import org.apache.accumulo.core.tabletserver.thrift.ScanState;
 +import org.apache.accumulo.core.tabletserver.thrift.ScanType;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 +import org.apache.accumulo.core.util.ByteBufferUtil;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.ColumnFQ;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.MapCounter;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ServerServices.Service;
 +import org.apache.accumulo.core.util.SimpleThreadPool;
 +import org.apache.accumulo.core.util.Stat;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
 +import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerConstants;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.ClientServiceHandler;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.data.ServerMutation;
 +import org.apache.accumulo.server.fs.FileRef;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManager.FileType;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.fs.VolumeUtil;
 +import org.apache.accumulo.server.log.SortedLogState;
 +import org.apache.accumulo.server.master.recovery.RecoveryPath;
 +import org.apache.accumulo.server.master.state.Assignment;
 +import org.apache.accumulo.server.master.state.DistributedStoreException;
 +import org.apache.accumulo.server.master.state.TServerInstance;
 +import org.apache.accumulo.server.master.state.TabletLocationState;
 +import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
 +import org.apache.accumulo.server.master.state.TabletStateStore;
 +import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 +import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
 +import org.apache.accumulo.server.problems.ProblemReport;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.FileSystemMonitor;
 +import org.apache.accumulo.server.util.Halt;
 +import org.apache.accumulo.server.util.MasterMetadataUtil;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.RpcWrapper;
 +import org.apache.accumulo.server.util.TServerUtils;
 +import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
 +import org.apache.accumulo.server.util.time.RelativeTime;
 +import org.apache.accumulo.server.util.time.SimpleTimer;
 +import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
 +import org.apache.accumulo.server.zookeeper.TransactionWatcher;
 +import org.apache.accumulo.server.zookeeper.ZooCache;
 +import org.apache.accumulo.server.zookeeper.ZooLock;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.accumulo.start.classloader.vfs.ContextManager;
 +import org.apache.accumulo.trace.instrument.Span;
 +import org.apache.accumulo.trace.instrument.Trace;
 +import org.apache.accumulo.trace.thrift.TInfo;
 +import org.apache.accumulo.tserver.Compactor.CompactionInfo;
 +import org.apache.accumulo.tserver.RowLocks.RowLock;
 +import org.apache.accumulo.tserver.Tablet.CommitSession;
 +import org.apache.accumulo.tserver.Tablet.KVEntry;
 +import org.apache.accumulo.tserver.Tablet.LookupResult;
 +import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
 +import org.apache.accumulo.tserver.Tablet.ScanBatch;
 +import org.apache.accumulo.tserver.Tablet.Scanner;
 +import org.apache.accumulo.tserver.Tablet.SplitInfo;
 +import org.apache.accumulo.tserver.Tablet.TConstraintViolationException;
 +import org.apache.accumulo.tserver.Tablet.TabletClosedException;
 +import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
 +import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
 +import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
 +import org.apache.accumulo.tserver.data.ServerConditionalMutation;
 +import org.apache.accumulo.tserver.log.DfsLogger;
 +import org.apache.accumulo.tserver.log.LogSorter;
 +import org.apache.accumulo.tserver.log.MutationReceiver;
 +import org.apache.accumulo.tserver.log.TabletServerLogger;
 +import org.apache.accumulo.tserver.mastermessage.MasterMessage;
 +import org.apache.accumulo.tserver.mastermessage.SplitReportMessage;
 +import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
 +import org.apache.accumulo.tserver.metrics.TabletServerMBean;
 +import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
 +import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
 +import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
 +import org.apache.commons.collections.map.LRUMap;
 +import org.apache.hadoop.fs.FSError;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.TException;
 +import org.apache.thrift.TProcessor;
 +import org.apache.thrift.TServiceClient;
 +import org.apache.thrift.server.TServer;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.NoNodeException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +enum ScanRunState {
 +  QUEUED, RUNNING, FINISHED
 +}
 +
 +public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.tserver.metrics.TabletServerMBean {
 +  private static final Logger log = Logger.getLogger(TabletServer.class);
 +
 +  private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
 +  private static long lastMemorySize = 0;
 +  private static long lastMemoryCheckTime = 0;
 +  private static long gcTimeIncreasedCount;
 +
 +  private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
 +  private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000;
 +  private static final long TIME_BETWEEN_GC_CHECKS = 5000;
 +
 +  private TabletServerLogger logger;
 +
 +  protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
 +
 +  private ServerConfiguration serverConfig;
 +  private LogSorter logSorter = null;
 +
 +  public TabletServer(ServerConfiguration conf, VolumeManager fs) {
 +    super();
 +    this.serverConfig = conf;
 +    this.instance = conf.getInstance();
 +    this.fs = fs;
 +    this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
 +    SimpleTimer.getInstance().schedule(new Runnable() {
 +      @Override
 +      public void run() {
 +        synchronized (onlineTablets) {
 +          long now = System.currentTimeMillis();
 +          for (Tablet tablet : onlineTablets.values())
 +            try {
 +              tablet.updateRates(now);
 +            } catch (Exception ex) {
 +              log.error(ex, ex);
 +            }
 +        }
 +      }
 +    }, TIME_BETWEEN_GC_CHECKS, TIME_BETWEEN_GC_CHECKS);
 +  }
 +
 +  private synchronized static void logGCInfo(AccumuloConfiguration conf) {
 +    long now = System.currentTimeMillis();
 +
 +    List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
 +    Runtime rt = Runtime.getRuntime();
 +
 +    StringBuilder sb = new StringBuilder("gc");
 +
 +    boolean sawChange = false;
 +
 +    long maxIncreaseInCollectionTime = 0;
 +
 +    for (GarbageCollectorMXBean gcBean : gcmBeans) {
 +      Long prevTime = prevGcTime.get(gcBean.getName());
 +      long pt = 0;
 +      if (prevTime != null) {
 +        pt = prevTime;
 +      }
 +
 +      long time = gcBean.getCollectionTime();
 +
 +      if (time - pt != 0) {
 +        sawChange = true;
 +      }
 +
 +      long increaseInCollectionTime = time - pt;
 +      sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
 +      maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
 +      prevGcTime.put(gcBean.getName(), time);
 +    }
 +
 +    long mem = rt.freeMemory();
 +    if (maxIncreaseInCollectionTime == 0) {
 +      gcTimeIncreasedCount = 0;
 +    } else {
 +      gcTimeIncreasedCount++;
 +      if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
 +        log.warn("Running low on memory");
 +        gcTimeIncreasedCount = 0;
 +      }
 +    }
 +
 +    if (mem > lastMemorySize) {
 +      sawChange = true;
 +    }
 +
 +    String sign = "+";
 +    if (mem - lastMemorySize <= 0) {
 +      sign = "";
 +    }
 +
 +    sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
 +
 +    if (sawChange) {
 +      log.debug(sb.toString());
 +    }
 +
 +    final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
 +    if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) {
 +      long diff = now - lastMemoryCheckTime;
 +      if (diff > keepAliveTimeout) {
 +        log.warn(String.format("GC pause checker not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check",
 +                    TIME_BETWEEN_GC_CHECKS / 1000., diff / 1000.));
 +      }
 +      lastMemoryCheckTime = now;
 +      return;
 +    }
 +
 +    if (maxIncreaseInCollectionTime > keepAliveTimeout) {
 +      Halt.halt("Garbage collection may be interfering with lock keep-alive.  Halting.", -1);
 +    }
 +
 +    lastMemorySize = mem;
 +    lastMemoryCheckTime = now;
 +  }
 +
 +  private TabletStatsKeeper statsKeeper;
 +
 +  private static class Session {
 +    long lastAccessTime;
 +    long startTime;
 +    String user;
 +    String client = TServerUtils.clientAddress.get();
 +    public boolean reserved;
 +
 +    public void cleanup() {}
 +  }
 +
 +  private static class SessionManager {
 +
 +    SecureRandom random;
 +    Map<Long,Session> sessions;
 +    long maxIdle;
 +
 +    SessionManager(AccumuloConfiguration conf) {
 +      random = new SecureRandom();
 +      sessions = new HashMap<Long,Session>();
 +
 +      maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
 +
 +      Runnable r = new Runnable() {
 +        @Override
 +        public void run() {
 +          sweep(maxIdle);
 +        }
 +      };
 +
 +      SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
 +    }
 +
 +    synchronized long createSession(Session session, boolean reserve) {
 +      long sid = random.nextLong();
 +
 +      while (sessions.containsKey(sid)) {
 +        sid = random.nextLong();
 +      }
 +
 +      sessions.put(sid, session);
 +
 +      session.reserved = reserve;
 +
 +      session.startTime = session.lastAccessTime = System.currentTimeMillis();
 +
 +      return sid;
 +    }
 +
 +    long getMaxIdleTime() {
 +      return maxIdle;
 +    }
 +
 +    /**
 +     * while a session is reserved, it cannot be canceled or removed
 +     */
 +    synchronized Session reserveSession(long sessionId) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null) {
 +        if (session.reserved)
 +          throw new IllegalStateException();
 +        session.reserved = true;
 +      }
 +
 +      return session;
 +
 +    }
 +
 +    synchronized Session reserveSession(long sessionId, boolean wait) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null) {
 +        while (wait && session.reserved) {
 +          try {
 +            wait(1000);
 +          } catch (InterruptedException e) {
 +            throw new RuntimeException();
 +          }
 +        }
 +
 +        if (session.reserved)
 +          throw new IllegalStateException();
 +        session.reserved = true;
 +      }
 +
 +      return session;
 +
 +    }
 +
 +    synchronized void unreserveSession(Session session) {
 +      if (!session.reserved)
 +        throw new IllegalStateException();
 +      notifyAll();
 +      session.reserved = false;
 +      session.lastAccessTime = System.currentTimeMillis();
 +    }
 +
 +    synchronized void unreserveSession(long sessionId) {
 +      Session session = getSession(sessionId);
 +      if (session != null)
 +        unreserveSession(session);
 +    }
 +
 +    synchronized Session getSession(long sessionId) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null)
 +        session.lastAccessTime = System.currentTimeMillis();
 +      return session;
 +    }
 +
 +    Session removeSession(long sessionId) {
 +      return removeSession(sessionId, false);
 +    }
 +
 +    Session removeSession(long sessionId, boolean unreserve) {
 +      Session session = null;
 +      synchronized (this) {
 +        session = sessions.remove(sessionId);
 +        if (unreserve && session != null)
 +          unreserveSession(session);
 +      }
 +
 +      // do clean up out side of lock..
 +      if (session != null)
 +        session.cleanup();
 +
 +      return session;
 +    }
 +
 +    private void sweep(long maxIdle) {
 +      ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
 +      synchronized (this) {
 +        Iterator<Session> iter = sessions.values().iterator();
 +        while (iter.hasNext()) {
 +          Session session = iter.next();
 +          long idleTime = System.currentTimeMillis() - session.lastAccessTime;
 +          if (idleTime > maxIdle && !session.reserved) {
 +            iter.remove();
 +            sessionsToCleanup.add(session);
 +          }
 +        }
 +      }
 +
 +      // do clean up outside of lock
 +      for (Session session : sessionsToCleanup) {
 +        session.cleanup();
 +      }
 +    }
 +
 +    synchronized void removeIfNotAccessed(final long sessionId, long delay) {
 +      Session session = sessions.get(sessionId);
 +      if (session != null) {
 +        final long removeTime = session.lastAccessTime;
 +        TimerTask r = new TimerTask() {
 +          @Override
 +          public void run() {
 +            Session sessionToCleanup = null;
 +            synchronized (SessionManager.this) {
 +              Session session2 = sessions.get(sessionId);
 +              if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
 +                sessions.remove(sessionId);
 +                sessionToCleanup = session2;
 +              }
 +            }
 +
 +            // call clean up outside of lock
 +            if (sessionToCleanup != null)
 +              sessionToCleanup.cleanup();
 +          }
 +        };
 +
 +        SimpleTimer.getInstance().schedule(r, delay);
 +      }
 +    }
 +
 +    public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
 +      Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
 +      for (Entry<Long,Session> entry : sessions.entrySet()) {
 +
 +        Session session = entry.getValue();
 +        @SuppressWarnings("rawtypes")
 +        ScanTask nbt = null;
 +        String tableID = null;
 +
 +        if (session instanceof ScanSession) {
 +          ScanSession ss = (ScanSession) session;
 +          nbt = ss.nextBatchTask;
 +          tableID = ss.extent.getTableId().toString();
 +        } else if (session instanceof MultiScanSession) {
 +          MultiScanSession mss = (MultiScanSession) session;
 +          nbt = mss.lookupTask;
 +          tableID = mss.threadPoolExtent.getTableId().toString();
 +        }
 +
 +        if (nbt == null)
 +          continue;
 +
 +        ScanRunState srs = nbt.getScanRunState();
 +
 +        if (srs == ScanRunState.FINISHED)
 +          continue;
 +
 +        MapCounter<ScanRunState> stateCounts = counts.get(tableID);
 +        if (stateCounts == null) {
 +          stateCounts = new MapCounter<ScanRunState>();
 +          counts.put(tableID, stateCounts);
 +        }
 +
 +        stateCounts.increment(srs, 1);
 +      }
 +
 +      return counts;
 +    }
 +
 +    public synchronized List<ActiveScan> getActiveScans() {
 +
 +      ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
 +
 +      long ct = System.currentTimeMillis();
 +
 +      for (Entry<Long,Session> entry : sessions.entrySet()) {
 +        Session session = entry.getValue();
 +        if (session instanceof ScanSession) {
 +          ScanSession ss = (ScanSession) session;
 +
 +          ScanState state = ScanState.RUNNING;
 +
 +          ScanTask<ScanBatch> nbt = ss.nextBatchTask;
 +          if (nbt == null) {
 +            state = ScanState.IDLE;
 +          } else {
 +            switch (nbt.getScanRunState()) {
 +              case QUEUED:
 +                state = ScanState.QUEUED;
 +                break;
 +              case FINISHED:
 +                state = ScanState.IDLE;
 +                break;
 +              case RUNNING:
 +              default:
 +                /* do nothing */
 +                break;
 +            }
 +          }
 +
 +          activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
 +              state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
 +
 +        } else if (session instanceof MultiScanSession) {
 +          MultiScanSession mss = (MultiScanSession) session;
 +
 +          ScanState state = ScanState.RUNNING;
 +
 +          ScanTask<MultiScanResult> nbt = mss.lookupTask;
 +          if (nbt == null) {
 +            state = ScanState.IDLE;
 +          } else {
 +            switch (nbt.getScanRunState()) {
 +              case QUEUED:
 +                state = ScanState.QUEUED;
 +                break;
 +              case FINISHED:
 +                state = ScanState.IDLE;
 +                break;
 +              case RUNNING:
 +              default:
 +                /* do nothing */
 +                break;
 +            }
 +          }
 +
 +          activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
 +              ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
 +                  .getAuthorizationsBB()));
 +        }
 +      }
 +
 +      return activeScans;
 +    }
 +  }
 +
 +  static class TservConstraintEnv implements Environment {
 +
 +    private TCredentials credentials;
 +    private SecurityOperation security;
 +    private Authorizations auths;
 +    private KeyExtent ke;
 +
 +    TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
 +      this.security = secOp;
 +      this.credentials = credentials;
 +    }
 +
 +    void setExtent(KeyExtent ke) {
 +      this.ke = ke;
 +    }
 +
 +    @Override
 +    public KeyExtent getExtent() {
 +      return ke;
 +    }
 +
 +    @Override
 +    public String getUser() {
 +      return credentials.getPrincipal();
 +    }
 +
 +    @Override
 +    @Deprecated
 +    public Authorizations getAuthorizations() {
 +      if (auths == null)
 +        try {
 +          this.auths = security.getUserAuthorizations(credentials);
 +        } catch (ThriftSecurityException e) {
 +          throw new RuntimeException(e);
 +        }
 +      return auths;
 +    }
 +
 +    @Override
 +    public AuthorizationContainer getAuthorizationsContainer() {
 +      return new AuthorizationContainer() {
 +        @Override
 +        public boolean contains(ByteSequence auth) {
 +          try {
 +            return security.userHasAuthorizations(credentials,
 +                Collections.<ByteBuffer> singletonList(ByteBuffer.wrap(auth.getBackingArray(), auth.offset(), auth.length())));
 +          } catch (ThriftSecurityException e) {
 +            throw new RuntimeException(e);
 +          }
 +        }
 +      };
 +    }
 +  }
 +
 +  private abstract class ScanTask<T> implements RunnableFuture<T> {
 +
 +    protected AtomicBoolean interruptFlag;
 +    protected ArrayBlockingQueue<Object> resultQueue;
 +    protected AtomicInteger state;
 +    protected AtomicReference<ScanRunState> runState;
 +
 +    private static final int INITIAL = 1;
 +    private static final int ADDED = 2;
 +    private static final int CANCELED = 3;
 +
 +    ScanTask() {
 +      interruptFlag = new AtomicBoolean(false);
 +      runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
 +      state = new AtomicInteger(INITIAL);
 +      resultQueue = new ArrayBlockingQueue<Object>(1);
 +    }
 +
 +    protected void addResult(Object o) {
 +      if (state.compareAndSet(INITIAL, ADDED))
 +        resultQueue.add(o);
 +      else if (state.get() == ADDED)
 +        throw new IllegalStateException("Tried to add more than one result");
 +    }
 +
 +    @Override
 +    public boolean cancel(boolean mayInterruptIfRunning) {
 +      if (!mayInterruptIfRunning)
 +        throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
 +
 +      if (state.get() == CANCELED)
 +        return true;
 +
 +      if (state.compareAndSet(INITIAL, CANCELED)) {
 +        interruptFlag.set(true);
 +        resultQueue = null;
 +        return true;
 +      }
 +
 +      return false;
 +    }
 +
 +    @Override
 +    public T get() throws InterruptedException, ExecutionException {
 +      throw new UnsupportedOperationException();
 +    }
 +
 +    @SuppressWarnings("unchecked")
 +    @Override
 +    public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
 +
 +      ArrayBlockingQueue<Object> localRQ = resultQueue;
 +
 +      if (state.get() == CANCELED)
 +        throw new CancellationException();
 +
 +      if (localRQ == null && state.get() == ADDED)
 +        throw new IllegalStateException("Tried to get result twice");
 +
 +      Object r = localRQ.poll(timeout, unit);
 +
 +      // could have been canceled while waiting
 +      if (state.get() == CANCELED) {
 +        if (r != null)
 +          throw new IllegalStateException("Nothing should have been added when in canceled state");
 +
 +        throw new CancellationException();
 +      }
 +
 +      if (r == null)
 +        throw new TimeoutException();
 +
 +      // make this method stop working now that something is being
 +      // returned
 +      resultQueue = null;
 +
 +      if (r instanceof Throwable)
 +        throw new ExecutionException((Throwable) r);
 +
 +      return (T) r;
 +    }
 +
 +    @Override
 +    public boolean isCancelled() {
 +      return state.get() == CANCELED;
 +    }
 +
 +    @Override
 +    public boolean isDone() {
 +      return runState.get().equals(ScanRunState.FINISHED);
 +    }
 +
 +    public ScanRunState getScanRunState() {
 +      return runState.get();
 +    }
 +
 +  }
 +
 +  private static class ConditionalSession extends Session {
 +    public TCredentials credentials;
 +    public Authorizations auths;
 +    public String tableId;
 +    public AtomicBoolean interruptFlag;
 +
 +    @Override
 +    public void cleanup() {
 +      interruptFlag.set(true);
 +    }
 +  }
 +
 +  private static class UpdateSession extends Session {
 +    public Tablet currentTablet;
 +    public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
 +    Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
 +    HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
 +    public Violations violations;
 +    public TCredentials credentials;
 +    public long totalUpdates = 0;
 +    public long flushTime = 0;
 +    Stat prepareTimes = new Stat();
 +    Stat walogTimes = new Stat();
 +    Stat commitTimes = new Stat();
 +    Stat authTimes = new Stat();
 +    public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
 +    public long queuedMutationSize = 0;
 +    TservConstraintEnv cenv = null;
 +  }
 +
 +  private static class ScanSession extends Session {
 +    public KeyExtent extent;
 +    public HashSet<Column> columnSet;
 +    public List<IterInfo> ssiList;
 +    public Map<String,Map<String,String>> ssio;
 +    public Authorizations auths;
 +    public long entriesReturned = 0;
 +    public Stat nbTimes = new Stat();
 +    public long batchCount = 0;
 +    public volatile ScanTask<ScanBatch> nextBatchTask;
 +    public AtomicBoolean interruptFlag;
 +    public Scanner scanner;
 +    public long readaheadThreshold = Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
 +
 +    @Override
 +    public void cleanup() {
 +      try {
 +        if (nextBatchTask != null)
 +          nextBatchTask.cancel(true);
 +      } finally {
 +        if (scanner != null)
 +          scanner.close();
 +      }
 +    }
 +
 +  }
 +
 +  private static class MultiScanSession extends Session {
 +    HashSet<Column> columnSet;
 +    Map<KeyExtent,List<Range>> queries;
 +    public List<IterInfo> ssiList;
 +    public Map<String,Map<String,String>> ssio;
 +    public Authorizations auths;
 +
 +    // stats
 +    int numRanges;
 +    int numTablets;
 +    int numEntries;
 +    long totalLookupTime;
 +
 +    public volatile ScanTask<MultiScanResult> lookupTask;
 +    public KeyExtent threadPoolExtent;
 +
 +    @Override
 +    public void cleanup() {
 +      if (lookupTask != null)
 +        lookupTask.cancel(true);
 +    }
 +  }
 +
 +  /**
 +   * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
 +   * are monotonically increasing.
 +   *
 +   */
 +  static class WriteTracker {
 +    private static AtomicLong operationCounter = new AtomicLong(1);
 +    private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
 +
 +    WriteTracker() {
 +      for (TabletType ttype : TabletType.values()) {
 +        inProgressWrites.put(ttype, new TreeSet<Long>());
 +      }
 +    }
 +
 +    synchronized long startWrite(TabletType ttype) {
 +      long operationId = operationCounter.getAndIncrement();
 +      inProgressWrites.get(ttype).add(operationId);
 +      return operationId;
 +    }
 +
 +    synchronized void finishWrite(long operationId) {
 +      if (operationId == -1)
 +        return;
 +
 +      boolean removed = false;
 +
 +      for (TabletType ttype : TabletType.values()) {
 +        removed = inProgressWrites.get(ttype).remove(operationId);
 +        if (removed)
 +          break;
 +      }
 +
 +      if (!removed) {
 +        throw new IllegalArgumentException("Attempted to finish write not in progress,  operationId " + operationId);
 +      }
 +
 +      this.notifyAll();
 +    }
 +
 +    synchronized void waitForWrites(TabletType ttype) {
 +      long operationId = operationCounter.getAndIncrement();
 +      while (inProgressWrites.get(ttype).floor(operationId) != null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {
 +          log.error(e, e);
 +        }
 +      }
 +    }
 +
 +    public long startWrite(Set<Tablet> keySet) {
 +      if (keySet.size() == 0)
 +        return -1;
 +
 +      ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
 +
 +      for (Tablet tablet : keySet)
 +        extents.add(tablet.getExtent());
 +
 +      return startWrite(TabletType.type(extents));
 +    }
 +  }
 +
 +  public AccumuloConfiguration getSystemConfiguration() {
 +    return serverConfig.getConfiguration();
 +  }
 +
 +  TransactionWatcher watcher = new TransactionWatcher();
 +
 +  private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
 +
 +    SessionManager sessionManager;
 +
 +    AccumuloConfiguration acuConf = getSystemConfiguration();
 +
 +    TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
 +
 +    TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
 +
 +    WriteTracker writeTracker = new WriteTracker();
 +
 +    private RowLocks rowLocks = new RowLocks();
 +
 +    ThriftClientHandler() {
 +      super(instance, watcher, fs);
 +      log.debug(ThriftClientHandler.class.getName() + " created");
 +      sessionManager = new SessionManager(getSystemConfiguration());
 +      // Register the metrics MBean
 +      try {
 +        updateMetrics.register();
 +        scanMetrics.register();
 +      } catch (Exception e) {
 +        log.error("Exception registering MBean with MBean Server", e);
 +      }
 +    }
 +
 +    @Override
 +    public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
 +        throws ThriftSecurityException {
 +
 +      if (!security.canPerformSystemActions(credentials))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
 +
 +      for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
 +        TKeyExtent tke = entry.getKey();
 +        Map<String,MapFileInfo> fileMap = entry.getValue();
 +        Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
 +        for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
 +          Path path = new Path(mapping.getKey());
 +          FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
 +          path = ns.makeQualified(path);
 +          fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
 +        }
 +
 +        Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
 +
 +        if (importTablet == null) {
 +          failures.add(tke);
 +        } else {
 +          try {
 +            importTablet.importMapFiles(tid, fileRefMap, setTime);
 +          } catch (IOException ioe) {
 +            log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
 +            failures.add(tke);
 +          }
 +        }
 +      }
 +      return failures;
 +    }
 +
 +    private class NextBatchTask extends ScanTask<ScanBatch> {
 +
 +      private long scanID;
 +
 +      NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
 +        this.scanID = scanID;
 +        this.interruptFlag = interruptFlag;
 +
 +        if (interruptFlag.get())
 +          cancel(true);
 +      }
 +
 +      @Override
 +      public void run() {
 +
 +        final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
 +        String oldThreadName = Thread.currentThread().getName();
 +
 +        try {
 +          if (isCancelled() || scanSession == null)
 +            return;
 +
 +          runState.set(ScanRunState.RUNNING);
 +
 +          Thread.currentThread().setName(
 +              "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
 +
 +          Tablet tablet = onlineTablets.get(scanSession.extent);
 +
 +          if (tablet == null) {
 +            addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
 +            return;
 +          }
 +
 +          long t1 = System.currentTimeMillis();
 +          ScanBatch batch = scanSession.scanner.read();
 +          long t2 = System.currentTimeMillis();
 +          scanSession.nbTimes.addStat(t2 - t1);
 +
 +          // there should only be one thing on the queue at a time, so
 +          // it should be ok to call add()
 +          // instead of put()... if add() fails because queue is at
 +          // capacity it means there is code
 +          // problem somewhere
 +          addResult(batch);
 +        } catch (TabletClosedException e) {
 +          addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
 +        } catch (IterationInterruptedException iie) {
 +          if (!isCancelled()) {
 +            log.warn("Iteration interrupted, when scan not cancelled", iie);
 +            addResult(iie);
 +          }
 +        } catch (TooManyFilesException tmfe) {
 +          addResult(tmfe);
 +        } catch (Throwable e) {
 +          log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
 +          addResult(e);
 +        } finally {
 +          runState.set(ScanRunState.FINISHED);
 +          Thread.currentThread().setName(oldThreadName);
 +        }
 +
 +      }
 +    }
 +
 +    private class LookupTask extends ScanTask<MultiScanResult> {
 +
 +      private long scanID;
 +
 +      LookupTask(long scanID) {
 +        this.scanID = scanID;
 +      }
 +
 +      @Override
 +      public void run() {
 +        MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
 +        String oldThreadName = Thread.currentThread().getName();
 +
 +        try {
 +          if (isCancelled() || session == null)
 +            return;
 +
 +          TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
 +          long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
 +
 +          runState.set(ScanRunState.RUNNING);
 +          Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
 +
 +          long bytesAdded = 0;
 +          long maxScanTime = 4000;
 +
 +          long startTime = System.currentTimeMillis();
 +
 +          ArrayList<KVEntry> results = new ArrayList<KVEntry>();
 +          Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
 +          ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
 +          KeyExtent partScan = null;
 +          Key partNextKey = null;
 +          boolean partNextKeyInclusive = false;
 +
 +          Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
 +
 +          // check the time so that the read ahead thread is not monopolized
 +          while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
 +            Entry<KeyExtent,List<Range>> entry = iter.next();
 +
 +            iter.remove();
 +
 +            // check that tablet server is serving requested tablet
 +            Tablet tablet = onlineTablets.get(entry.getKey());
 +            if (tablet == null) {
 +              failures.put(entry.getKey(), entry.getValue());
 +              continue;
 +            }
 +            Thread.currentThread().setName(
 +                "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
 +
 +            LookupResult lookupResult;
 +            try {
 +
 +              // do the following check to avoid a race condition
 +              // between setting false below and the task being
 +              // canceled
 +              if (isCancelled())
 +                interruptFlag.set(true);
 +
 +              lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
 +                  session.ssio, interruptFlag);
 +
 +              // if the tablet was closed it it possible that the
 +              // interrupt flag was set.... do not want it set for
 +              // the next
 +              // lookup
 +              interruptFlag.set(false);
 +
 +            } catch (IOException e) {
 +              log.warn("lookup failed for tablet " + entry.getKey(), e);
 +              throw new RuntimeException(e);
 +            }
 +
 +            bytesAdded += lookupResult.bytesAdded;
 +
 +            if (lookupResult.unfinishedRanges.size() > 0) {
 +              if (lookupResult.closed) {
 +                failures.put(entry.getKey(), lookupResult.unfinishedRanges);
 +              } else {
 +                session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
 +                partScan = entry.getKey();
 +                partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
 +                partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
 +              }
 +            } else {
 +              fullScans.add(entry.getKey());
 +            }
 +          }
 +
 +          long finishTime = System.currentTimeMillis();
 +          session.totalLookupTime += (finishTime - startTime);
 +          session.numEntries += results.size();
 +
 +          // convert everything to thrift before adding result
 +          List<TKeyValue> retResults = new ArrayList<TKeyValue>();
 +          for (KVEntry entry : results)
 +            retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
 +          Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
 +          List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
 +          TKeyExtent retPartScan = null;
 +          TKey retPartNextKey = null;
 +          if (partScan != null) {
 +            retPartScan = partScan.toThrift();
 +            retPartNextKey = partNextKey.toThrift();
 +          }
 +          // add results to queue
 +          addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
 +        } catch (IterationInterruptedException iie) {
 +          if (!isCancelled()) {
 +            log.warn("Iteration interrupted, when scan not cancelled", iie);
 +            addResult(iie);
 +          }
 +        } catch (Throwable e) {
 +          log.warn("exception while doing multi-scan ", e);
 +          addResult(e);
 +        } finally {
 +          Thread.currentThread().setName(oldThreadName);
 +          runState.set(ScanRunState.FINISHED);
 +        }
 +      }
 +    }
 +
 +    @Override
 +    public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
 +        List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
 +        long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
 +
-       String tableId = new String(textent.getTable(), Constants.UTF8);
++      String tableId = new String(textent.getTable(), UTF_8);
 +      if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      if (!security.userHasAuthorizations(credentials, authorizations))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
 +
 +      KeyExtent extent = new KeyExtent(textent);
 +
 +      // wait for any writes that are in flight.. this done to ensure
 +      // consistency across client restarts... assume a client writes
 +      // to accumulo and dies while waiting for a confirmation from
 +      // accumulo... the client process restarts and tries to read
 +      // data from accumulo making the assumption that it will get
 +      // any writes previously made... however if the server side thread
 +      // processing the write from the dead client is still in progress,
 +      // the restarted client may not see the write unless we wait here.
 +      // this behavior is very important when the client is reading the
 +      // metadata
 +      if (waitForWrites)
 +        writeTracker.waitForWrites(TabletType.type(extent));
 +
 +      Tablet tablet = onlineTablets.get(extent);
 +      if (tablet == null)
 +        throw new NotServingTabletException(textent);
 +
 +      ScanSession scanSession = new ScanSession();
 +      scanSession.user = credentials.getPrincipal();
 +      scanSession.extent = new KeyExtent(extent);
 +      scanSession.columnSet = new HashSet<Column>();
 +      scanSession.ssiList = ssiList;
 +      scanSession.ssio = ssio;
 +      scanSession.auths = new Authorizations(authorizations);
 +      scanSession.interruptFlag = new AtomicBoolean();
 +      scanSession.readaheadThreshold = readaheadThreshold;
 +
 +      for (TColumn tcolumn : columns) {
 +        scanSession.columnSet.add(new Column(tcolumn));
 +      }
 +
 +      scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
 +          scanSession.interruptFlag);
 +
 +      long sid = sessionManager.createSession(scanSession, true);
 +
 +      ScanResult scanResult;
 +      try {
 +        scanResult = continueScan(tinfo, sid, scanSession);
 +      } catch (NoSuchScanIDException e) {
 +        log.error("The impossible happened", e);
 +        throw new RuntimeException();
 +      } finally {
 +        sessionManager.unreserveSession(sid);
 +      }
 +
 +      return new InitialScan(sid, scanResult);
 +    }
 +
 +    @Override
 +    public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
 +        org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
 +      ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
 +      if (scanSession == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      try {
 +        return continueScan(tinfo, scanID, scanSession);
 +      } finally {
 +        sessionManager.unreserveSession(scanSession);
 +      }
 +    }
 +
 +    private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
 +        org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
 +
 +      if (scanSession.nextBatchTask == null) {
 +        scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
 +        resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
 +      }
 +
 +      ScanBatch bresult;
 +      try {
 +        bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
 +        scanSession.nextBatchTask = null;
 +      } catch (ExecutionException e) {
 +        sessionManager.removeSession(scanID);
 +        if (e.getCause() instanceof NotServingTabletException)
 +          throw (NotServingTabletException) e.getCause();
 +        else if (e.getCause() instanceof TooManyFilesException)
 +          throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
 +        else
 +          throw new RuntimeException(e);
 +      } catch (CancellationException ce) {
 +        sessionManager.removeSession(scanID);
 +        Tablet tablet = onlineTablets.get(scanSession.extent);
 +        if (tablet == null || tablet.isClosed())
 +          throw new NotServingTabletException(scanSession.extent.toThrift());
 +        else
 +          throw new NoSuchScanIDException();
 +      } catch (TimeoutException e) {
 +        List<TKeyValue> param = Collections.emptyList();
 +        long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
 +        sessionManager.removeIfNotAccessed(scanID, timeout);
 +        return new ScanResult(param, true);
 +      } catch (Throwable t) {
 +        sessionManager.removeSession(scanID);
 +        log.warn("Failed to get next batch", t);
 +        throw new RuntimeException(t);
 +      }
 +
 +      ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
 +
 +      scanSession.entriesReturned += scanResult.results.size();
 +
 +      scanSession.batchCount++;
 +
 +      if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
 +        // start reading next batch while current batch is transmitted
 +        // to client
 +        scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
 +        resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
 +      }
 +
 +      if (!scanResult.more)
 +        closeScan(tinfo, scanID);
 +
 +      return scanResult;
 +    }
 +
 +    @Override
 +    public void closeScan(TInfo tinfo, long scanID) {
 +      ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
 +      if (ss != null) {
 +        long t2 = System.currentTimeMillis();
 +
 +        log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
 +            .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
 +        if (scanMetrics.isEnabled()) {
 +          scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
 +          scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
 +        }
 +      }
 +    }
 +
 +    @Override
 +    public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
 +        List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
 +      // find all of the tables that need to be scanned
 +      HashSet<String> tables = new HashSet<String>();
 +      for (TKeyExtent keyExtent : tbatch.keySet()) {
-         tables.add(new String(keyExtent.getTable(), Constants.UTF8));
++        tables.add(new String(keyExtent.getTable(), UTF_8));
 +      }
 +
 +      if (tables.size() != 1)
 +        throw new IllegalArgumentException("Cannot batch scan over multiple tables");
 +
 +      // check if user has permission to the tables
 +      for (String tableId : tables)
 +        if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), tbatch, tcolumns, ssiList, ssio, authorizations))
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      try {
 +        if (!security.userHasAuthorizations(credentials, authorizations))
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
 +      } catch (ThriftSecurityException tse) {
 +        log.error(tse, tse);
 +        throw tse;
 +      }
 +      Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, new TKeyExtentTranslator(), new Translator.ListTranslator<TRange,Range>(
 +          new TRangeTranslator()));
 +
 +      // This is used to determine which thread pool to use
 +      KeyExtent threadPoolExtent = batch.keySet().iterator().next();
 +
 +      if (waitForWrites)
 +        writeTracker.waitForWrites(TabletType.type(batch.keySet()));
 +
 +      MultiScanSession mss = new MultiScanSession();
 +      mss.user = credentials.getPrincipal();
 +      mss.queries = batch;
 +      mss.columnSet = new HashSet<Column>(tcolumns.size());
 +      mss.ssiList = ssiList;
 +      mss.ssio = ssio;
 +      mss.auths = new Authorizations(authorizations);
 +
 +      mss.numTablets = batch.size();
 +      for (List<Range> ranges : batch.values()) {
 +        mss.numRanges += ranges.size();
 +      }
 +
 +      for (TColumn tcolumn : tcolumns)
 +        mss.columnSet.add(new Column(tcolumn));
 +
 +      mss.threadPoolExtent = threadPoolExtent;
 +
 +      long sid = sessionManager.createSession(mss, true);
 +
 +      MultiScanResult result;
 +      try {
 +        result = continueMultiScan(tinfo, sid, mss);
 +      } catch (NoSuchScanIDException e) {
 +        log.error("the impossible happened", e);
 +        throw new RuntimeException("the impossible happened", e);
 +      } finally {
 +        sessionManager.unreserveSession(sid);
 +      }
 +
 +      return new InitialMultiScan(sid, result);
 +    }
 +
 +    @Override
 +    public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
 +
 +      MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
 +
 +      if (session == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      try {
 +        return continueMultiScan(tinfo, scanID, session);
 +      } finally {
 +        sessionManager.unreserveSession(session);
 +      }
 +    }
 +
 +    private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
 +
 +      if (session.lookupTask == null) {
 +        session.lookupTask = new LookupTask(scanID);
 +        resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
 +      }
 +
 +      try {
 +        MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
 +        session.lookupTask = null;
 +        return scanResult;
 +      } catch (TimeoutException e1) {
 +        long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
 +        sessionManager.removeIfNotAccessed(scanID, timeout);
 +        List<TKeyValue> results = Collections.emptyList();
 +        Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
 +        List<TKeyExtent> fullScans = Collections.emptyList();
 +        return new MultiScanResult(results, failures, fullScans, null, null, false, true);
 +      } catch (Throwable t) {
 +        sessionManager.removeSession(scanID);
 +        log.warn("Failed to get multiscan result", t);
 +        throw new RuntimeException(t);
 +      }
 +    }
 +
 +    @Override
 +    public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
 +      MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
 +      if (session == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      long t2 = System.currentTimeMillis();
 +      log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
 +          session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
 +    }
 +
 +    @Override
 +    public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
 +      // Make sure user is real
 +
 +      security.authenticateUser(credentials, credentials);
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
 +
 +      UpdateSession us = new UpdateSession();
 +      us.violations = new Violations();
 +      us.credentials = credentials;
 +      us.cenv = new TservConstraintEnv(security, us.credentials);
 +
 +      long sid = sessionManager.createSession(us, false);
 +
 +      return sid;
 +    }
 +
 +    private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
 +      long t1 = System.currentTimeMillis();
 +      if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
 +        return;
 +      if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
 +        // if there were previous failures, then do not accept additional writes
 +        return;
 +      }
 +
 +      try {
 +        // if user has no permission to write to this table, add it to
 +        // the failures list
 +        boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
 +        String tableId = keyExtent.getTableId().toString();
 +        if (sameTable || security.canWrite(us.credentials, tableId, Tables.getNamespaceId(instance, tableId))) {
 +          long t2 = System.currentTimeMillis();
 +          us.authTimes.addStat(t2 - t1);
 +          us.currentTablet = onlineTablets.get(keyExtent);
 +          if (us.currentTablet != null) {
 +            us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
 +          } else {
 +            // not serving tablet, so report all mutations as
 +            // failures
 +            us.failures.put(keyExtent, 0l);
 +            if (updateMetrics.isEnabled())
 +              updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
 +          }
 +        } else {
 +          log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
 +          long t2 = System.currentTimeMillis();
 +          us.authTimes.addStat(t2 - t1);
 +          us.currentTablet = null;
 +          us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
 +          if (updateMetrics.isEnabled())
 +            updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
 +          return;
 +        }
 +      } catch (ThriftSecurityException e) {
 +        log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
 +        long t2 = System.currentTimeMillis();
 +        us.authTimes.addStat(t2 - t1);
 +        us.currentTablet = null;
 +        us.authFailures.put(keyExtent, e.getCode());
 +        if (updateMetrics.isEnabled())
 +          updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
 +        return;
 +      }
 +    }
 +
 +    @Override
 +    public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
 +      UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
 +      if (us == null) {
 +        throw new RuntimeException("No Such SessionID");
 +      }
 +
 +      try {
 +        KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +        setUpdateTablet(us, keyExtent);
 +
 +        if (us.currentTablet != null) {
 +          List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
 +          for (TMutation tmutation : tmutations) {
 +            Mutation mutation = new ServerMutation(tmutation);
 +            mutations.add(mutation);
 +            us.queuedMutationSize += mutation.numBytes();
 +          }
 +          if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
 +            flush(us);
 +        }
 +      } finally {
 +        sessionManager.unreserveSession(us);
 +      }
 +    }
 +
 +    private void flush(UpdateSession us) {
 +
 +      int mutationCount = 0;
 +      Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
 +      Throwable error = null;
 +
 +      long pt1 = System.currentTimeMillis();
 +
 +      boolean containsMetadataTablet = false;
 +      for (Tablet tablet : us.queuedMutations.keySet())
 +        if (tablet.getExtent().isMeta())
 +          containsMetadataTablet = true;
 +
 +      if (!containsMetadataTablet && us.queuedMutations.size() > 0)
 +        TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
 +
 +      Span prep = Trace.start("prep");
 +      try {
 +        for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
 +
 +          Tablet tablet = entry.getKey();
 +          List<Mutation> mutations = entry.getValue();
 +          if (mutations.size() > 0) {
 +            try {
 +              if (updateMetrics.isEnabled())
 +                updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
 +
 +              CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
 +              if (commitSession == null) {
 +                if (us.currentTablet == tablet) {
 +                  us.currentTablet = null;
 +                }
 +                us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
 +              } else {
 +                sendables.put(commitSession, mutations);
 +                mutationCount += mutations.size();
 +              }
 +
 +            } catch (TConstraintViolationException e) {
 +              us.violations.add(e.getViolations());
 +              if (updateMetrics.isEnabled())
 +                updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
 +
 +              if (e.getNonViolators().size() > 0) {
 +                // only log and commit mutations if there were some
 +                // that did not
 +                // violate constraints... this is what
 +                // prepareMutationsForCommit()
 +                // expects
 +                sendables.put(e.getCommitSession(), e.getNonViolators());
 +              }
 +
 +              mutationCount += mutations.size();
 +
 +            } catch (HoldTimeoutException t) {
 +              error = t;
 +              log.debug("Giving up on mutations due to a long memory hold time");
 +              break;
 +            } catch (Throwable t) {
 +              error = t;
 +              log.error("Unexpected error preparing for commit", error);
 +              break;
 +            }
 +          }
 +        }
 +      } finally {
 +        prep.stop();
 +      }
 +
 +      long pt2 = System.currentTimeMillis();
 +      us.prepareTimes.addStat(pt2 - pt1);
 +      updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
 +
 +      if (error != null) {
 +        for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
 +          e.getKey().abortCommit(e.getValue());
 +        }
 +        throw new RuntimeException(error);
 +      }
 +      try {
 +        Span wal = Trace.start("wal");
 +        try {
 +          while (true) {
 +            try {
 +              long t1 = System.currentTimeMillis();
 +
 +              logger.logManyTablets(sendables);
 +
 +              long t2 = System.currentTimeMillis();
 +              us.walogTimes.addStat(t2 - t1);
 +              updateWalogWriteTime((t2 - t1));
 +              break;
 +            } catch (IOException ex) {
 +              log.warn("logging mutations failed, retrying");
 +            } catch (FSError ex) { // happens when DFS is localFS
 +              log.warn("logging mutations failed, retrying");
 +            } catch (Throwable t) {
 +              log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
 +              throw new RuntimeException(t);
 +            }
 +          }
 +        } finally {
 +          wal.stop();
 +        }
 +
 +        Span commit = Trace.start("commit");
 +        try {
 +          long t1 = System.currentTimeMillis();
 +          for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
 +            CommitSession commitSession = entry.getKey();
 +            List<Mutation> mutations = entry.getValue();
 +
 +            commitSession.commit(mutations);
 +
 +            Tablet tablet = commitSession.getTablet();
 +
 +            if (tablet == us.currentTablet) {
 +              // because constraint violations may filter out some
 +              // mutations, for proper
 +              // accounting with the client code, need to increment
 +              // the count based
 +              // on the original number of mutations from the client
 +              // NOT the filtered number
 +              us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
 +            }
 +          }
 +          long t2 = System.currentTimeMillis();
 +
 +          us.flushTime += (t2 - pt1);
 +          us.commitTimes.addStat(t2 - t1);
 +
 +          updateAvgCommitTime(t2 - t1, sendables.size());
 +        } finally {
 +          commit.stop();
 +        }
 +      } finally {
 +        us.queuedMutations.clear();
 +        if (us.currentTablet != null) {
 +          us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
 +        }
 +        us.queuedMutationSize = 0;
 +      }
 +      us.totalUpdates += mutationCount;
 +    }
 +
 +    private void updateWalogWriteTime(long time) {
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, time);
 +    }
 +
 +    private void updateAvgCommitTime(long time, int size) {
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.commitTime, (long) ((time) / (double) size));
 +    }
 +
 +    private void updateAvgPrepTime(long time, int size) {
 +      if (updateMetrics.isEnabled())
 +        updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (long) ((time) / (double) size));
 +    }
 +
 +    @Override
 +    public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
 +      UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
 +      if (us == null) {
 +        throw new NoSuchScanIDException();
 +      }
 +
 +      // clients may or may not see data from an update session while
 +      // it is in progress, however when the update session is closed
 +      // want to ensure that reads wait for the write to finish
 +      long opid = writeTracker.startWrite(us.queuedMutations.keySet());
 +
 +      try {
 +        flush(us);
 +      } finally {
 +        writeTracker.finishWrite(opid);
 +      }
 +
 +      log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
 +          (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
 +          us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
 +      if (us.failures.size() > 0) {
 +        Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
 +        log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
 +      }
 +      List<ConstraintViolationSummary> violations = us.violations.asList();
 +      if (violations.size() > 0) {
 +        ConstraintViolationSummary first = us.violations.asList().iterator().next();
 +        log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
 +      }
 +      if (us.authFailures.size() > 0) {
 +        KeyExtent first = us.authFailures.keySet().iterator().next();
 +        log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
 +      }
 +
 +      return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
 +          us.authFailures, Translators.KET));
 +    }
 +
 +    @Override
 +    public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
 +        ConstraintViolationException, ThriftSecurityException {
 +
-       String tableId = new String(tkeyExtent.getTable(), Constants.UTF8);
++      String tableId = new String(tkeyExtent.getTable(), UTF_8);
 +      if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +      KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +      Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
 +      if (tablet == null) {
 +        throw new NotServingTabletException(tkeyExtent);
 +      }
 +
 +      if (!keyExtent.isMeta())
 +        TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
 +
 +      long opid = writeTracker.startWrite(TabletType.type(keyExtent));
 +
 +      try {
 +        Mutation mutation = new ServerMutation(tmutation);
 +        List<Mutation> mutations = Collections.singletonList(mutation);
 +
 +        Span prep = Trace.start("prep");
 +        CommitSession cs;
 +        try {
 +          cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
 +        } finally {
 +          prep.stop();
 +        }
 +        if (cs == null) {
 +          throw new NotServingTabletException(tkeyExtent);
 +        }
 +
 +        while (true) {
 +          try {
 +            Span wal = Trace.start("wal");
 +            try {
 +              logger.log(cs, cs.getWALogSeq(), mutation);
 +            } finally {
 +              wal.stop();
 +            }
 +            break;
 +          } catch (IOException ex) {
 +            log.warn(ex, ex);
 +          }
 +        }
 +
 +        Span commit = Trace.start("commit");
 +        try {
 +          cs.commit(mutations);
 +        } finally {
 +          commit.stop();
 +        }
 +      } catch (TConstraintViolationException e) {
 +        throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
 +      } finally {
 +        writeTracker.finishWrite(opid);
 +      }
 +    }
 +
 +    private void checkConditions(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs,
 +        List<String> symbols) throws IOException {
 +      Iterator<Entry<KeyExtent,List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
 +
 +      CompressedIterators compressedIters = new CompressedIterators(symbols);
 +
 +      while (iter.hasNext()) {
 +        Entry<KeyExtent,List<ServerConditionalMutation>> entry = iter.next();
 +        Tablet tablet = onlineTablets.get(entry.getKey());
 +
 +        if (tablet == null || tablet.isClosed()) {
 +          for (ServerConditionalMutation scm : entry.getValue())
 +            results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          iter.remove();
 +        } else {
 +          List<ServerConditionalMutation> okMutations = new ArrayList<ServerConditionalMutation>(entry.getValue().size());
 +
 +          for (ServerConditionalMutation scm : entry.getValue()) {
 +            if (checkCondition(results, cs, compressedIters, tablet, scm))
 +              okMutations.add(scm);
 +          }
 +
 +          entry.setValue(okMutations);
 +        }
 +
 +      }
 +    }
 +
 +    boolean checkCondition(ArrayList<TCMResult> results, ConditionalSession cs, CompressedIterators compressedIters, Tablet tablet,
 +        ServerConditionalMutation scm) throws IOException {
 +      boolean add = true;
 +
 +      Set<Column> emptyCols = Collections.emptySet();
 +
 +      for (TCondition tc : scm.getConditions()) {
 +
 +        Range range;
 +        if (tc.hasTimestamp)
 +          range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()), tc.getTs());
 +        else
 +          range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()));
 +
 +        IterConfig ic = compressedIters.decompress(tc.iterators);
 +
 +        Scanner scanner = tablet.createScanner(range, 1, emptyCols, cs.auths, ic.ssiList, ic.ssio, false, cs.interruptFlag);
 +
 +        try {
 +          ScanBatch batch = scanner.read();
 +
 +          Value val = null;
 +
 +          for (KVEntry entry2 : batch.results) {
 +            val = entry2.getValue();
 +            break;
 +          }
 +
 +          if ((val == null ^ tc.getVal() == null) || (val != null && !Arrays.equals(tc.getVal(), val.get()))) {
 +            results.add(new TCMResult(scm.getID(), TCMStatus.REJECTED));
 +            add = false;
 +            break;
 +          }
 +
 +        } catch (TabletClosedException e) {
 +          results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          add = false;
 +          break;
 +        } catch (IterationInterruptedException iie) {
 +          results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          add = false;
 +          break;
 +        } catch (TooManyFilesException tmfe) {
 +          results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          add = false;
 +          break;
 +        }
 +      }
 +      return add;
 +    }
 +
 +    private void writeConditionalMutations(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
 +      Set<Entry<KeyExtent,List<ServerConditionalMutation>>> es = updates.entrySet();
 +
 +      Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
 +
 +      boolean sessionCanceled = sess.interruptFlag.get();
 +
 +      Span prepSpan = Trace.start("prep");
 +      try {
 +        long t1 = System.currentTimeMillis();
 +        for (Entry<KeyExtent,List<ServerConditionalMutation>> entry : es) {
 +          Tablet tablet = onlineTablets.get(entry.getKey());
 +          if (tablet == null || tablet.isClosed() || sessionCanceled) {
 +            for (ServerConditionalMutation scm : entry.getValue())
 +              results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +          } else {
 +            try {
 +
 +              @SuppressWarnings("unchecked")
 +              List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
 +              if (mutations.size() > 0) {
 +
 +                CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, sess.credentials), mutations);
 +
 +                if (cs == null) {
 +                  for (ServerConditionalMutation scm : entry.getValue())
 +                    results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
 +                } else {
 +                  for (ServerConditionalMutation scm : entry.getValue())
 +                    results.add(new TCMResult(scm.getID(), TCMStatus.ACCEPTED));
 +                  sendables.put(cs, mutations);
 +                }
 +              }
 +            } catch (TConstraintViolationException e) {
 +              if (e.getNonViolators().size() > 0) {
 +                sendables.put(e.getCommitSession(), e.getNonViolators());
 +                for (Mutation m : e.getNonViolators())
 +                  results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.ACCEPTED));
 +              }
 +
 +              for (Mutation m : e.getViolators())
 +                results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.VIOLATED));
 +            }
 +          }
 +        }
 +
 +        long t2 = System.currentTimeMillis();
 +        updateAvgPrepTime(t2 - t1, es.size());
 +      } finally {
 +        prepSpan.stop();
 +      }
 +
 +      Span walSpan = Trace.start("wal");
 +      try {
 +        while (true && sendables.size() > 0) {
 +          try {
 +            long t1 = System.currentTimeMillis();
 +            logger.logManyTablets(sendables);
 +            long t2 = System.currentTimeMillis();
 +            updateWalogWriteTime(t2 - t1);
 +            break;
 +          } catch (IOException ex) {
 +            log.warn("logging mutations failed, retrying");
 +          } catch (FSError ex) { // happens when DFS is localFS
 +            log.warn("logging mutations failed, retrying");
 +          } catch (Throwable t) {
 +            log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
 +            throw new RuntimeException(t);
 +          }
 +        }
 +      } finally {
 +        walSpan.stop();
 +      }
 +
 +      Span commitSpan = Trace.start("commit");
 +      try {
 +        long t1 = System.currentTimeMillis();
 +        for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
 +          CommitSession commitSession = entry.getKey();
 +          List<Mutation> mutations = entry.getValue();
 +
 +          commitSession.commit(mutations);
 +        }
 +        long t2 = System.currentTimeMillis();
 +        updateAvgCommitTime(t2 - t1, sendables.size());
 +      } finally {
 +        commitSpan.stop();
 +      }
 +
 +    }
 +
 +    private Map<KeyExtent,List<ServerConditionalMutation>> conditionalUpdate(ConditionalSession cs, Map<KeyExtent,List<ServerConditionalMutation>> updates,
 +        ArrayList<TCMResult> results, List<String> symbols) throws IOException {
 +      // sort each list of mutations, this is done to avoid deadlock and doing seeks in order is more efficient and detect duplicate rows.
 +      ConditionalMutationSet.sortConditionalMutations(updates);
 +
 +      Map<KeyExtent,List<ServerConditionalMutation>> deferred = new HashMap<KeyExtent,List<ServerConditionalMutation>>();
 +
 +      // can not process two mutations for the same row, because one will not see what the other writes
 +      ConditionalMutationSet.deferDuplicatesRows(updates, deferred);
 +
 +      // get as many locks as possible w/o blocking... defer any rows that are locked
 +      List<RowLock> locks = rowLocks.acquireRowlocks(updates, deferred);
 +      try {
 +        Span checkSpan = Trace.start("Check conditions");
 +        try {
 +          checkConditions(updates, results, cs, symbols);
 +        } finally {
 +          checkSpan.stop();
 +        }
 +
 +        Span updateSpan = Trace.start("apply conditional mutations");
 +        try {
 +          writeConditionalMutations(updates, results, cs);
 +        } finally {
 +          updateSpan.stop();
 +        }
 +      } finally {
 +        rowLocks.releaseRowLocks(locks);
 +      }
 +      return deferred;
 +    }
 +
 +    @Override
 +    public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials credentials, List<ByteBuffer> authorizations, String tableId)
 +        throws ThriftSecurityException, TException {
 +
 +      Authorizations userauths = null;
 +      if (!security.canConditionallyUpdate(credentials, tableId, Tables.getNamespaceId(instance, tableId), authorizations))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      userauths = security.getUserAuthorizations(credentials);
 +      for (ByteBuffer auth : authorizations)
 +        if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
 +
 +      ConditionalSession cs = new ConditionalSession();
 +      cs.auths = new Authorizations(authorizations);
 +      cs.credentials = credentials;
 +      cs.tableId = tableId;
 +      cs.interruptFlag = new AtomicBoolean();
 +
 +      long sid = sessionManager.createSession(cs, false);
 +      return new TConditionalSession(sid, lockID, sessionManager.getMaxIdleTime());
 +    }
 +
 +    @Override
 +    public List<TCMResult> conditionalUpdate(TInfo tinfo, long sessID, Map<TKeyExtent,List<TConditionalMutation>> mutations, List<String> symbols)
 +        throws NoSuchScanIDException, TException {
 +
 +      ConditionalSession cs = (ConditionalSession) sessionManager.reserveSession(sessID);
 +
 +      if (cs == null || cs.interruptFlag.get())
 +        throw new NoSuchScanIDException();
 +
 +      if (!cs.tableId.equals(MetadataTable.ID) && !cs.tableId.equals(RootTable.ID))
 +        TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
 +
 +      Text tid = new Text(cs.tableId);
 +      long opid = writeTracker.startWrite(TabletType.type(new KeyExtent(tid, null, null)));
 +
 +      try {
 +        Map<KeyExtent,List<ServerConditionalMutation>> updates = Translator.translate(mutations, Translators.TKET,
 +            new Translator.ListTranslator<TConditionalMutation,ServerConditionalMutation>(ServerConditionalMutation.TCMT));
 +
 +        for (KeyExtent ke : updates.keySet())
 +          if (!ke.getTableId().equals(tid))
 +            throw new IllegalArgumentException("Unexpected table id " + tid + " != " + ke.getTableId());
 +
 +        ArrayList<TCMResult> results = new ArrayList<TCMResult>();
 +
 +        Map<KeyExtent,List<ServerConditionalMutation>> deferred = conditionalUpdate(cs, updates, results, symbols);
 +
 +        while (deferred.size() > 0) {
 +          deferred = conditionalUpdate(cs, deferred, results, symbols);
 +        }
 +
 +        return results;
 +      } catch (IOException ioe) {
 +        throw new TException(ioe);
 +      } finally {
 +        writeTracker.finishWrite(opid);
 +        sessionManager.unreserveSession(sessID);
 +      }
 +    }
 +
 +    @Override
 +    public void invalidateConditionalUpdate(TInfo tinfo, long sessID) throws TException {
 +      // this method should wait for any running conditional update to complete
 +      // after this method returns a conditional update should not be able to start
 +
 +      ConditionalSession cs = (ConditionalSession) sessionManager.getSession(sessID);
 +      if (cs != null)
 +        cs.interruptFlag.set(true);
 +
 +      cs = (ConditionalSession) sessionManager.reserveSession(sessID, true);
 +      if (cs != null)
 +        sessionManager.removeSession(sessID, true);
 +    }
 +
 +    @Override
 +    public void closeConditionalUpdate(TInfo tinfo, long sessID) throws TException {
 +      sessionManager.removeSession(sessID, false);
 +    }
 +
 +    @Override
 +    public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
 +        ThriftSecurityException {
 +
 +      String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table));
 +      String namespaceId;
 +      try {
 +        namespaceId = Tables.getNamespaceId(instance, tableId);
 +      } catch (IllegalArgumentException ex) {
 +        // table does not exist, try to educate the client
 +        throw new NotServingTabletException(tkeyExtent);
 +      }
 +
 +      if (!security.canSplitTablet(credentials, tableId, namespaceId))
 +        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +
 +      KeyExtent keyExtent = new KeyExtent(tkeyExtent);
 +
 +      Tablet tablet = onlineTablets.get(keyExtent);
 +      if (tablet == null) {
 +        throw new NotServingTabletException(tkeyExtent);
 +      }
 +
 +      if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
 +        try {
 +          if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
 +            throw new NotServingTabletException(tkeyExtent);
 +          }
 +        } catch (IOException e) {
 +          log.warn("Failed to split " + keyExtent, e);
 +          throw new RuntimeException(e);
 +        }
 +      }
 +    }
 +
 +    @Override
 +    public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
 +      return getStats(sessionManager.getActiveScansPerTable());
 +    }
 +
 +    @Override
 +    public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
 +      TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
 +      synchronized (onlineTablets) {
 +        onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
 +      }
 +      List<TabletStats> result = new ArrayList<TabletStats>();
 +      Text text = new Text(tableId);
 +      KeyExtent start = new KeyExtent(text, new Text(), null);
 +      for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
 +        KeyExtent ke = entry.getKey();
 +        if (ke.getTableId().compareTo(text) == 0) {
 +          Tablet tablet = entry.getValue();
 +          TabletStats stats = tablet.timer.getTabletStats();
 +          stats.extent = ke.toThrift();
 +          stats.ingestRate = tablet.ingestRate();
 +          stats.queryRate = tablet.queryRate();
 +          stats.splitCreationTime = tablet.getSplitCreationTime();
 +          stats.numEntries = tablet.getNumEntries();
 +          result.add(stats);
 +        }
 +      }
 +      return result;
 +    }
 +
 +    private ZooCache masterLockCache = new ZooCache();
 +
 +    private void checkPermission(TCredentials credentials, String lock, final String request) throws ThriftSecurityException {
 +      boolean fatal = false;
 +      try {
 +        log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
 +        if (!security.canPerformSystemActions(credentials)) {
 +          log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
 +          throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
 +        }
 +      } catch (ThriftSecurityException e) {
 +        log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
 +        if (SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName())) {
 +          log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configu

<TRUNCATED>