You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2014/11/01 05:56:55 UTC
[01/50] [abbrv] Merge branch '1.5' into 1.6
Repository: accumulo
Updated Branches:
refs/heads/1.5 69d665cb4 -> 2deabd31a
refs/heads/1.6 5e0f6a426 -> 9b20a9d40
refs/heads/master 64d1bbe9b -> cfbdef134
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
index c07ed1c,0000000..d77d060
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
@@@ -1,193 -1,0 +1,194 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LargeRowIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 2, MemoryUnit.BYTE);
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "10ms"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private static final int SEED = 42;
+ private static final String REG_TABLE_NAME = "lr";
+ private static final String PRE_SPLIT_TABLE_NAME = "lrps";
+ private static final int NUM_ROWS = 100;
+ private static final int ROW_SIZE = 1 << 17;
+ private static final int NUM_PRE_SPLITS = 9;
+ private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / NUM_PRE_SPLITS;
+
+ private int timeoutFactor = 1;
+
+ @Before
+ public void getTimeoutFactor() {
+ try {
+ timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
+ } catch (NumberFormatException e) {
+ log.warn("Could not parse property value for 'timeout.factor' as integer: " + System.getProperty("timeout.factor"));
+ }
+
+ Assert.assertTrue("Timeout factor must be greater than or equal to 1", timeoutFactor >= 1);
+ }
+
+ @Test
+ public void run() throws Exception {
+ Random r = new Random();
+ byte rowData[] = new byte[ROW_SIZE];
+ r.setSeed(SEED + 1);
+ TreeSet<Text> splitPoints = new TreeSet<Text>();
+ for (int i = 0; i < NUM_PRE_SPLITS; i++) {
+ r.nextBytes(rowData);
+ TestIngest.toPrintableChars(rowData);
+ splitPoints.add(new Text(rowData));
+ }
+ Connector c = getConnector();
+ c.tableOperations().create(REG_TABLE_NAME);
+ c.tableOperations().create(PRE_SPLIT_TABLE_NAME);
+ c.tableOperations().addSplits(PRE_SPLIT_TABLE_NAME, splitPoints);
+ test1(c);
+ test2(c);
+ }
+
+ private void test1(Connector c) throws Exception {
+
+ basicTest(c, REG_TABLE_NAME, 0);
+
+ c.tableOperations().setProperty(REG_TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "" + SPLIT_THRESH);
+
+ UtilWaitThread.sleep(timeoutFactor * 12000);
+ Logger.getLogger(LargeRowIT.class).warn("checking splits");
+ FunctionalTestUtils.checkSplits(c, REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
+
+ verify(c, REG_TABLE_NAME);
+ }
+
+ private void test2(Connector c) throws Exception {
+ basicTest(c, PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
+ }
+
+ private void basicTest(Connector c, String table, int expectedSplits) throws Exception {
+ BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
+
+ Random r = new Random();
+ byte rowData[] = new byte[ROW_SIZE];
+
+ r.setSeed(SEED);
+
+ for (int i = 0; i < NUM_ROWS; i++) {
+
+ r.nextBytes(rowData);
+ TestIngest.toPrintableChars(rowData);
+
+ Mutation mut = new Mutation(new Text(rowData));
- mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(UTF_8)));
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+
+ verify(c, table);
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+
+ c.tableOperations().flush(table, null, null, false);
+
+ // verify while table flush is running
+ verify(c, table);
+
+ // give split time to complete
+ c.tableOperations().flush(table, null, null, true);
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+
+ verify(c, table);
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+ }
+
+ private void verify(Connector c, String table) throws Exception {
+ Random r = new Random();
+ byte rowData[] = new byte[ROW_SIZE];
+
+ r.setSeed(SEED);
+
+ Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
+
+ for (int i = 0; i < NUM_ROWS; i++) {
+
+ r.nextBytes(rowData);
+ TestIngest.toPrintableChars(rowData);
+
+ scanner.setRange(new Range(new Text(rowData)));
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(new Text(rowData))) {
+ throw new Exception("verification failed, unexpected row i =" + i);
+ }
- if (!entry.getValue().equals(Integer.toString(i).getBytes(Constants.UTF8))) {
++ if (!entry.getValue().equals(Integer.toString(i).getBytes(UTF_8))) {
+ throw new Exception("verification failed, unexpected value i =" + i + " value = " + entry.getValue());
+ }
+ count++;
+ }
+
+ if (count != 1) {
+ throw new Exception("verification failed, unexpected count i =" + i + " count=" + count);
+ }
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 3102f59,0000000..f4bee97
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -1,612 -1,0 +1,613 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Random;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.tserver.NativeMap;
+import org.apache.hadoop.io.Text;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class NativeMapIT {
+
+ private Key nk(int r) {
+ return new Key(new Text(String.format("r%09d", r)));
+ }
+
+ private Key nk(int r, int cf, int cq, int cv, int ts, boolean deleted) {
+ Key k = new Key(new Text(String.format("r%09d", r)), new Text(String.format("cf%09d", cf)), new Text(String.format("cq%09d", cq)), new Text(String.format(
+ "cv%09d", cv)), ts);
+
+ k.setDeleted(deleted);
+
+ return k;
+ }
+
+ private Value nv(int v) {
- return new Value(String.format("r%09d", v).getBytes(Constants.UTF8));
++ return new Value(String.format("r%09d", v).getBytes(UTF_8));
+ }
+
+ public static File nativeMapLocation() {
+ File projectDir = new File(System.getProperty("user.dir")).getParentFile();
+ File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION);
+ return nativeMapDir;
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
+ }
+
+ private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {
+ for (int i = start; i <= end; i++) {
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ assertEquals(nk(i), entry.getKey());
+ assertEquals(nv(i + valueOffset), entry.getValue());
+ }
+
+ assertFalse(iter.hasNext());
+ }
+
+ private void insertAndVerify(NativeMap nm, int start, int end, int valueOffset) {
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i + valueOffset));
+ }
+
+ for (int i = start; i <= end; i++) {
+ Value v = nm.get(nk(i));
+ assertNotNull(v);
+ assertEquals(nv(i + valueOffset), v);
+
+ Iterator<Entry<Key,Value>> iter2 = nm.iterator(nk(i));
+ assertTrue(iter2.hasNext());
+ Entry<Key,Value> entry = iter2.next();
+ assertEquals(nk(i), entry.getKey());
+ assertEquals(nv(i + valueOffset), entry.getValue());
+ }
+
+ assertNull(nm.get(nk(start - 1)));
+
+ assertNull(nm.get(nk(end + 1)));
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ verifyIterator(start, end, valueOffset, iter);
+
+ for (int i = start; i <= end; i++) {
+ iter = nm.iterator(nk(i));
+ verifyIterator(i, end, valueOffset, iter);
+
+ // lookup nonexistant key that falls after existing key
+ iter = nm.iterator(nk(i, 1, 1, 1, 1, false));
+ verifyIterator(i + 1, end, valueOffset, iter);
+ }
+
+ assertEquals(end - start + 1, nm.size());
+ }
+
+ private void insertAndVerifyExhaustive(NativeMap nm, int num, int run) {
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = 0; ts < num; ts++) {
+ Key key = nk(i, j, k, l, ts, true);
- Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ nm.put(key, value);
+
+ key = nk(i, j, k, l, ts, false);
- value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ nm.put(key, value);
+ }
+ }
+ }
+ }
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = num - 1; ts >= 0; ts--) {
+ Key key = nk(i, j, k, l, ts, true);
- Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+
+ key = nk(i, j, k, l, ts, false);
- value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ assertTrue(iter.hasNext());
+ entry = iter.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ assertFalse(iter.hasNext());
+
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = 0; ts < num; ts++) {
+ Key key = nk(i, j, k, l, ts, true);
- Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ assertEquals(value, nm.get(key));
+
+ Iterator<Entry<Key,Value>> iter2 = nm.iterator(key);
+ assertTrue(iter2.hasNext());
+ Entry<Key,Value> entry = iter2.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+
+ key = nk(i, j, k, l, ts, false);
- value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ assertEquals(value, nm.get(key));
+
+ Iterator<Entry<Key,Value>> iter3 = nm.iterator(key);
+ assertTrue(iter3.hasNext());
+ Entry<Key,Value> entry2 = iter3.next();
+ assertEquals(key, entry2.getKey());
+ assertEquals(value, entry2.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ assertEquals(num * num * num * num * num * 2, nm.size());
+ }
+
+ @Test
+ public void test1() {
+ NativeMap nm = new NativeMap();
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ assertFalse(iter.hasNext());
+ nm.delete();
+ }
+
+ @Test
+ public void test2() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+ insertAndVerify(nm, 1, 10, 1);
+ insertAndVerify(nm, 1, 10, 2);
+
+ nm.delete();
+ }
+
+ @Test
+ public void test4() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerifyExhaustive(nm, 3, 0);
+ insertAndVerifyExhaustive(nm, 3, 1);
+
+ nm.delete();
+ }
+
+ @Test
+ public void test5() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ iter.next();
+
+ nm.delete();
+
+ try {
+ nm.put(nk(1), nv(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.get(nk(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.iterator();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.iterator(nk(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.size();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ }
+
+ @Test
+ public void test7() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+
+ nm.delete();
+
+ try {
+ nm.delete();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+ }
+
+ @Test
+ public void test8() {
+ // test verifies that native map sorts keys sharing some common prefix properly
+
+ NativeMap nm = new NativeMap();
+
+ TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
+
+ tm.put(new Key(new Text("fo")), new Value(new byte[] {'0'}));
+ tm.put(new Key(new Text("foo")), new Value(new byte[] {'1'}));
+ tm.put(new Key(new Text("foo1")), new Value(new byte[] {'2'}));
+ tm.put(new Key(new Text("foo2")), new Value(new byte[] {'3'}));
+
+ for (Entry<Key,Value> entry : tm.entrySet()) {
+ nm.put(entry.getKey(), entry.getValue());
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ for (Entry<Key,Value> entry : tm.entrySet()) {
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry2 = iter.next();
+
+ assertEquals(entry.getKey(), entry2.getKey());
+ assertEquals(entry.getValue(), entry2.getValue());
+ }
+
+ assertFalse(iter.hasNext());
+
+ nm.delete();
+ }
+
+ @Test
+ public void test9() {
+ NativeMap nm = new NativeMap();
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (NoSuchElementException e) {
+
+ }
+
+ insertAndVerify(nm, 1, 1, 0);
+
+ iter = nm.iterator();
+ iter.next();
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (NoSuchElementException e) {
+
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void test10() {
+ int start = 1;
+ int end = 10000;
+
+ NativeMap nm = new NativeMap();
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i));
+ }
+
+ long mem1 = nm.getMemoryUsed();
+
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i));
+ }
+
+ long mem2 = nm.getMemoryUsed();
+
+ if (mem1 != mem2) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem2);
+ }
+
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i));
+ }
+
+ long mem3 = nm.getMemoryUsed();
+
+ if (mem1 != mem3) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem3);
+ }
+
+ byte bigrow[] = new byte[1000000];
+ byte bigvalue[] = new byte[bigrow.length];
+
+ for (int i = 0; i < bigrow.length; i++) {
+ bigrow[i] = (byte) (0xff & (i % 256));
+ bigvalue[i] = bigrow[i];
+ }
+
+ nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
+
+ long mem4 = nm.getMemoryUsed();
+
+ Value val = nm.get(new Key(new Text(bigrow)));
+ if (val == null || !val.equals(new Value(bigvalue))) {
+ throw new RuntimeException("Did not get expected big value");
+ }
+
+ nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
+
+ long mem5 = nm.getMemoryUsed();
+
+ if (mem4 != mem5) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem4 + " " + mem5);
+ }
+
+ val = nm.get(new Key(new Text(bigrow)));
+ if (val == null || !val.equals(new Value(bigvalue))) {
+ throw new RuntimeException("Did not get expected big value");
+ }
+
+ nm.delete();
+ }
+
+ // random length random field
+ private static byte[] rlrf(Random r, int maxLen) {
+ int len = r.nextInt(maxLen);
+
+ byte f[] = new byte[len];
+ r.nextBytes(f);
+
+ return f;
+ }
+
+ @Test
+ public void test11() {
+ NativeMap nm = new NativeMap();
+
+ // insert things with varying field sizes and value sizes
+
+ // generate random data
+ Random r = new Random(75);
+
+ ArrayList<Pair<Key,Value>> testData = new ArrayList<Pair<Key,Value>>();
+
+ for (int i = 0; i < 100000; i++) {
+
+ Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false);
+ Value v = new Value(rlrf(r, 511));
+
+ testData.add(new Pair<Key,Value>(k, v));
+ }
+
+ // insert unsorted data
+ for (Pair<Key,Value> pair : testData) {
+ nm.put(pair.getFirst(), pair.getSecond());
+ }
+
+ for (int i = 0; i < 2; i++) {
+
+ // sort data
+ Collections.sort(testData, new Comparator<Pair<Key,Value>>() {
+ @Override
+ public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
+ return o1.getFirst().compareTo(o2.getFirst());
+ }
+ });
+
+ // verify
+ Iterator<Entry<Key,Value>> iter1 = nm.iterator();
+ Iterator<Pair<Key,Value>> iter2 = testData.iterator();
+
+ while (iter1.hasNext() && iter2.hasNext()) {
+ Entry<Key,Value> e = iter1.next();
+ Pair<Key,Value> p = iter2.next();
+
+ if (!e.getKey().equals(p.getFirst()))
+ throw new RuntimeException("Keys not equal");
+
+ if (!e.getValue().equals(p.getSecond()))
+ throw new RuntimeException("Values not equal");
+ }
+
+ if (iter1.hasNext())
+ throw new RuntimeException("Not all of native map consumed");
+
+ if (iter2.hasNext())
+ throw new RuntimeException("Not all of test data consumed");
+
+ System.out.println("test 11 nm mem " + nm.getMemoryUsed());
+
+ // insert data again w/ different value
+ Collections.shuffle(testData, r);
+ // insert unsorted data
+ for (Pair<Key,Value> pair : testData) {
+ pair.getSecond().set(rlrf(r, 511));
+ nm.put(pair.getFirst(), pair.getSecond());
+ }
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void testBinary() {
+ NativeMap nm = new NativeMap();
+
+ byte emptyBytes[] = new byte[0];
+
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ nm.put(k, v);
+ }
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+
+ assertEquals(k, entry.getKey());
+ assertEquals(v, entry.getValue());
+
+ }
+ }
+
+ assertFalse(iter.hasNext());
+
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ Value v2 = nm.get(k);
+
+ assertEquals(v, v2);
+ }
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void testEmpty() {
+ NativeMap nm = new NativeMap();
+
+ assertTrue(nm.size() == 0);
+ assertTrue(nm.getMemoryUsed() == 0);
+
+ nm.delete();
+ }
+
+ @Test
+ public void testConcurrentIter() throws IOException {
+ NativeMap nm = new NativeMap();
+
+ nm.put(nk(0), nv(0));
+ nm.put(nk(1), nv(1));
+ nm.put(nk(3), nv(3));
+
+ SortedKeyValueIterator<Key,Value> iter = nm.skvIterator();
+
+ // modify map after iter created
+ nm.put(nk(2), nv(2));
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(0));
+ iter.next();
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(1));
+ iter.next();
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(2));
+ iter.next();
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(3));
+ iter.next();
+
+ assertFalse(iter.hasTop());
+
+ nm.delete();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index 7e43812,0000000..189a55c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@@ -1,132 -1,0 +1,133 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ScanIteratorIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 30;
+ }
+
+ @Test
+ public void run() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 0; i < 1000; i++) {
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(Constants.UTF8)));
- m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(UTF_8)));
++ m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ Scanner scanner = c.createScanner(tableName, new Authorizations());
+
+ setupIter(scanner);
+ verify(scanner, 1, 999);
+
+ BatchScanner bscanner = c.createBatchScanner(tableName, new Authorizations(), 3);
+ bscanner.setRanges(Collections.singleton(new Range((Key) null, null)));
+
+ setupIter(bscanner);
+ verify(bscanner, 1, 999);
+
+ ArrayList<Range> ranges = new ArrayList<Range>();
+ ranges.add(new Range(new Text(String.format("%06d", 1))));
+ ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16))));
+ ranges.add(new Range(new Text(String.format("%06d", 20))));
+ ranges.add(new Range(new Text(String.format("%06d", 23))));
+ ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61))));
+ ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504))));
+ ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000))));
+
+ HashSet<Integer> got = new HashSet<Integer>();
+ HashSet<Integer> expected = new HashSet<Integer>();
+ for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) {
+ expected.add(i);
+ }
+
+ bscanner.setRanges(ranges);
+
+ for (Entry<Key,Value> entry : bscanner) {
+ got.add(Integer.parseInt(entry.getKey().getRow().toString()));
+ }
+
+ System.out.println("got : " + got);
+
+ if (!got.equals(expected)) {
+ throw new Exception(got + " != " + expected);
+ }
+
+ bscanner.close();
+
+ }
+
+ private void verify(Iterable<Entry<Key,Value>> scanner, int start, int finish) throws Exception {
+
+ int expected = start;
+ for (Entry<Key,Value> entry : scanner) {
+ if (Integer.parseInt(entry.getKey().getRow().toString()) != expected) {
+ throw new Exception("Saw unexpexted " + entry.getKey().getRow() + " " + expected);
+ }
+
+ if (entry.getKey().getColumnQualifier().toString().equals("cq2")) {
+ expected += 2;
+ }
+ }
+
+ if (expected != finish + 2) {
+ throw new Exception("Ended at " + expected + " not " + (finish + 2));
+ }
+ }
+
+ private void setupIter(ScannerBase scanner) throws Exception {
+ IteratorSetting dropMod = new IteratorSetting(50, "dropMod", "org.apache.accumulo.test.functional.DropModIter");
+ dropMod.addOption("mod", "2");
+ dropMod.addOption("drop", "0");
+ scanner.addScanIterator(dropMod);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
index ba03d00,0000000..90b881c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
@@@ -1,242 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ScanRangeIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ private static final int TS_LIMIT = 1;
+ private static final int CQ_LIMIT = 5;
+ private static final int CF_LIMIT = 5;
+ private static final int ROW_LIMIT = 100;
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = getUniqueNames(2);
+ String table1 = tableNames[0];
+ c.tableOperations().create(table1);
+ String table2 = tableNames[1];
+ c.tableOperations().create(table2);
+ TreeSet<Text> splitRows = new TreeSet<Text>();
+ int splits = 3;
+ for (int i = (ROW_LIMIT / splits); i < ROW_LIMIT; i += (ROW_LIMIT / splits))
+ splitRows.add(createRow(i));
+ c.tableOperations().addSplits(table2, splitRows);
+
+ insertData(c, table1);
+ scanTable(c, table1);
+
+ insertData(c, table2);
+ scanTable(c, table2);
+ }
+
+ private void scanTable(Connector c, String table) throws Exception {
+ scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(1, 0, 0, 0));
+
+ scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+
+ scanRange(c, table, null, null);
+
+ for (int i = 0; i < ROW_LIMIT; i += (ROW_LIMIT / 3)) {
+ for (int j = 0; j < CF_LIMIT; j += (CF_LIMIT / 2)) {
+ for (int k = 1; k < CQ_LIMIT; k += (CQ_LIMIT / 2)) {
+ scanRange(c, table, null, new IntKey(i, j, k, 0));
+ scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(i, j, k, 0));
+
+ scanRange(c, table, new IntKey(i, j, k, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+
+ scanRange(c, table, new IntKey(i, j, k, 0), null);
+
+ }
+ }
+ }
+
+ for (int i = 0; i < ROW_LIMIT; i++) {
+ scanRange(c, table, new IntKey(i, 0, 0, 0), new IntKey(i, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+
+ if (i > 0 && i < ROW_LIMIT - 1) {
+ scanRange(c, table, new IntKey(i - 1, 0, 0, 0), new IntKey(i + 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+ }
+ }
+
+ }
+
+ private static class IntKey {
+ private int row;
+ private int cf;
+ private int cq;
+ private long ts;
+
+ IntKey(IntKey ik) {
+ this.row = ik.row;
+ this.cf = ik.cf;
+ this.cq = ik.cq;
+ this.ts = ik.ts;
+ }
+
+ IntKey(int row, int cf, int cq, long ts) {
+ this.row = row;
+ this.cf = cf;
+ this.cq = cq;
+ this.ts = ts;
+ }
+
+ Key createKey() {
+ Text trow = createRow(row);
+ Text tcf = createCF(cf);
+ Text tcq = createCQ(cq);
+
+ return new Key(trow, tcf, tcq, ts);
+ }
+
+ IntKey increment() {
+
+ IntKey ik = new IntKey(this);
+
+ ik.ts++;
+ if (ik.ts >= TS_LIMIT) {
+ ik.ts = 0;
+ ik.cq++;
+ if (ik.cq >= CQ_LIMIT) {
+ ik.cq = 0;
+ ik.cf++;
+ if (ik.cf >= CF_LIMIT) {
+ ik.cf = 0;
+ ik.row++;
+ }
+ }
+ }
+
+ return ik;
+ }
+
+ }
+
+ private void scanRange(Connector c, String table, IntKey ik1, IntKey ik2) throws Exception {
+ scanRange(c, table, ik1, false, ik2, false);
+ scanRange(c, table, ik1, false, ik2, true);
+ scanRange(c, table, ik1, true, ik2, false);
+ scanRange(c, table, ik1, true, ik2, true);
+ }
+
+ private void scanRange(Connector c, String table, IntKey ik1, boolean inclusive1, IntKey ik2, boolean inclusive2) throws Exception {
+ Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
+
+ Key key1 = null;
+ Key key2 = null;
+
+ IntKey expectedIntKey;
+ IntKey expectedEndIntKey;
+
+ if (ik1 != null) {
+ key1 = ik1.createKey();
+ expectedIntKey = ik1;
+
+ if (!inclusive1) {
+ expectedIntKey = expectedIntKey.increment();
+ }
+ } else {
+ expectedIntKey = new IntKey(0, 0, 0, 0);
+ }
+
+ if (ik2 != null) {
+ key2 = ik2.createKey();
+ expectedEndIntKey = ik2;
+
+ if (inclusive2) {
+ expectedEndIntKey = expectedEndIntKey.increment();
+ }
+ } else {
+ expectedEndIntKey = new IntKey(ROW_LIMIT, 0, 0, 0);
+ }
+
+ Range range = new Range(key1, inclusive1, key2, inclusive2);
+
+ scanner.setRange(range);
+
+ for (Entry<Key,Value> entry : scanner) {
+
+ Key expectedKey = expectedIntKey.createKey();
+ if (!expectedKey.equals(entry.getKey())) {
+ throw new Exception(" " + expectedKey + " != " + entry.getKey());
+ }
+
+ expectedIntKey = expectedIntKey.increment();
+ }
+
+ if (!expectedIntKey.createKey().equals(expectedEndIntKey.createKey())) {
+ throw new Exception(" " + expectedIntKey.createKey() + " != " + expectedEndIntKey.createKey());
+ }
+ }
+
+ private static Text createCF(int cf) {
+ Text tcf = new Text(String.format("cf_%03d", cf));
+ return tcf;
+ }
+
+ private static Text createCQ(int cf) {
+ Text tcf = new Text(String.format("cq_%03d", cf));
+ return tcf;
+ }
+
+ private static Text createRow(int row) {
+ Text trow = new Text(String.format("r_%06d", row));
+ return trow;
+ }
+
+ private void insertData(Connector c, String table) throws Exception {
+
+ BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
+
+ for (int i = 0; i < ROW_LIMIT; i++) {
+ Mutation m = new Mutation(createRow(i));
+
+ for (int j = 0; j < CF_LIMIT; j++) {
+ for (int k = 0; k < CQ_LIMIT; k++) {
+ for (int t = 0; t < TS_LIMIT; t++) {
- m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(Constants.UTF8)));
++ m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(UTF_8)));
+ }
+ }
+ }
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
index ed2f0ba,0000000..3547b68
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
@@@ -1,111 -1,0 +1,112 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ScanSessionTimeOutIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_SESSION_MAXIDLE.getKey(), "3"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("abc");
+
+ BatchWriter bw = c.createBatchWriter("abc", new BatchWriterConfig());
+
+ for (int i = 0; i < 100000; i++) {
+ Mutation m = new Mutation(new Text(String.format("%08d", i)));
+ for (int j = 0; j < 3; j++)
- m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ Scanner scanner = c.createScanner("abc", new Authorizations());
+ scanner.setBatchSize(1000);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ verify(iter, 0, 200);
+
+ // sleep three times the session timeout
+ UtilWaitThread.sleep(9000);
+
+ verify(iter, 200, 100000);
+
+ }
+
+ private void verify(Iterator<Entry<Key,Value>> iter, int start, int stop) throws Exception {
+ for (int i = start; i < stop; i++) {
+
+ Text er = new Text(String.format("%08d", i));
+
+ for (int j = 0; j < 3; j++) {
+ Entry<Key,Value> entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(er)) {
+ throw new Exception("row " + entry.getKey().getRow() + " != " + er);
+ }
+
+ if (!entry.getKey().getColumnFamily().equals(new Text("cf1"))) {
+ throw new Exception("cf " + entry.getKey().getColumnFamily() + " != cf1");
+ }
+
+ if (!entry.getKey().getColumnQualifier().equals(new Text("cq" + j))) {
+ throw new Exception("cq " + entry.getKey().getColumnQualifier() + " != cq" + j);
+ }
+
+ if (!entry.getValue().toString().equals("" + i + "_" + j)) {
+ throw new Exception("value " + entry.getValue() + " != " + i + "_" + j);
+ }
+
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index ea92949,0000000..43d3a86
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@@ -1,272 -1,0 +1,273 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.util.FileUtil;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class SplitRecoveryIT extends ConfigurableMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ private KeyExtent nke(String table, String endRow, String prevEndRow) {
+ return new KeyExtent(new Text(table), endRow == null ? null : new Text(endRow), prevEndRow == null ? null : new Text(prevEndRow));
+ }
+
+ private void run() throws Exception {
+ String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/testLock";
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.OVERWRITE);
+ ZooLock zl = new ZooLock(zPath);
+ boolean gotLock = zl.tryLock(new LockWatcher() {
+
+ @Override
+ public void lostLock(LockLossReason reason) {
+ System.exit(-1);
+
+ }
+
+ @Override
+ public void unableToMonitorLockNode(Throwable e) {
+ System.exit(-1);
+ }
- }, "foo".getBytes(Constants.UTF8));
++ }, "foo".getBytes(UTF_8));
+
+ if (!gotLock) {
+ System.err.println("Failed to get lock " + zPath);
+ }
+
+ // run test for a table with one tablet
+ runSplitRecoveryTest(0, "sp", 0, zl, nke("foo0", null, null));
+ runSplitRecoveryTest(1, "sp", 0, zl, nke("foo1", null, null));
+
+ // run test for tables with two tablets, run test on first and last tablet
+ runSplitRecoveryTest(0, "k", 0, zl, nke("foo2", "m", null), nke("foo2", null, "m"));
+ runSplitRecoveryTest(1, "k", 0, zl, nke("foo3", "m", null), nke("foo3", null, "m"));
+ runSplitRecoveryTest(0, "o", 1, zl, nke("foo4", "m", null), nke("foo4", null, "m"));
+ runSplitRecoveryTest(1, "o", 1, zl, nke("foo5", "m", null), nke("foo5", null, "m"));
+
+ // run test for table w/ three tablets, run test on middle tablet
+ runSplitRecoveryTest(0, "o", 1, zl, nke("foo6", "m", null), nke("foo6", "r", "m"), nke("foo6", null, "r"));
+ runSplitRecoveryTest(1, "o", 1, zl, nke("foo7", "m", null), nke("foo7", "r", "m"), nke("foo7", null, "r"));
+
+ // run test for table w/ three tablets, run test on first
+ runSplitRecoveryTest(0, "g", 0, zl, nke("foo8", "m", null), nke("foo8", "r", "m"), nke("foo8", null, "r"));
+ runSplitRecoveryTest(1, "g", 0, zl, nke("foo9", "m", null), nke("foo9", "r", "m"), nke("foo9", null, "r"));
+
+ // run test for table w/ three tablets, run test on last tablet
+ runSplitRecoveryTest(0, "w", 2, zl, nke("fooa", "m", null), nke("fooa", "r", "m"), nke("fooa", null, "r"));
+ runSplitRecoveryTest(1, "w", 2, zl, nke("foob", "m", null), nke("foob", "r", "m"), nke("foob", null, "r"));
+ }
+
+ private void runSplitRecoveryTest(int failPoint, String mr, int extentToSplit, ZooLock zl, KeyExtent... extents) throws Exception {
+
+ Text midRow = new Text(mr);
+
+ SortedMap<FileRef,DataFileValue> splitMapFiles = null;
+
+ for (int i = 0; i < extents.length; i++) {
+ KeyExtent extent = extents[i];
+
+ String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString() + "/dir_" + i;
+ MetadataTableUtil.addTablet(extent, tdir, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID, zl);
+ SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>();
+ mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i));
+
+ if (i == extentToSplit) {
+ splitMapFiles = mapFiles;
+ }
+ int tid = 0;
+ TransactionWatcher.ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
+ MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles, "L0", SystemCredentials.get(), zl);
+ }
+
+ KeyExtent extent = extents[extentToSplit];
+
+ KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
+ KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
+
+ splitPartiallyAndRecover(extent, high, low, .4, splitMapFiles, midRow, "localhost:1234", failPoint, zl);
+ }
+
+ private void splitPartiallyAndRecover(KeyExtent extent, KeyExtent high, KeyExtent low, double splitRatio, SortedMap<FileRef,DataFileValue> mapFiles,
+ Text midRow, String location, int steps, ZooLock zl) throws Exception {
+
+ SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>();
+ SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
+ List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
+
+ MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, new HashMap<FileRef,FileUtil.FileInfo>(), mapFiles, lowDatafileSizes,
+ highDatafileSizes, highDatafilesToRemove);
+
+ MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, SystemCredentials.get(), zl);
+ TServerInstance instance = new TServerInstance(location, zl.getSessionId());
+ Writer writer = new Writer(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID);
+ Assignment assignment = new Assignment(high, instance);
+ Mutation m = new Mutation(assignment.tablet.getMetadataEntry());
+ m.put(TabletsSection.FutureLocationColumnFamily.NAME, assignment.server.asColumnQualifier(), assignment.server.asMutationValue());
+ writer.update(m);
+
+ if (steps >= 1) {
+ Map<FileRef,Long> bulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), extent);
+ MasterMetadataUtil.addNewTablet(low, "/lowDir", instance, lowDatafileSizes, bulkFiles, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID + "0", -1l,
+ -1l, zl);
+ }
+ if (steps >= 2)
+ MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SystemCredentials.get(), zl);
+
+ TabletServer.verifyTabletInformation(high, instance, null, "127.0.0.1:0", zl);
+
+ if (steps >= 1) {
+ ensureTabletHasNoUnexpectedMetadataEntries(low, lowDatafileSizes);
+ ensureTabletHasNoUnexpectedMetadataEntries(high, highDatafileSizes);
+
+ Map<FileRef,Long> lowBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), low);
+ Map<FileRef,Long> highBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), high);
+
+ if (!lowBulkFiles.equals(highBulkFiles)) {
+ throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
+ }
+
+ if (lowBulkFiles.size() == 0) {
+ throw new Exception(" no bulk files " + low);
+ }
+ } else {
+ ensureTabletHasNoUnexpectedMetadataEntries(extent, mapFiles);
+ }
+ }
+
+ private void ensureTabletHasNoUnexpectedMetadataEntries(KeyExtent extent, SortedMap<FileRef,DataFileValue> expectedMapFiles) throws Exception {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID, Authorizations.EMPTY);
+ scanner.setRange(extent.toMetadataRange());
+
+ HashSet<ColumnFQ> expectedColumns = new HashSet<ColumnFQ>();
+ expectedColumns.add(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN);
+ expectedColumns.add(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
+ expectedColumns.add(TabletsSection.ServerColumnFamily.TIME_COLUMN);
+ expectedColumns.add(TabletsSection.ServerColumnFamily.LOCK_COLUMN);
+
+ HashSet<Text> expectedColumnFamilies = new HashSet<Text>();
+ expectedColumnFamilies.add(DataFileColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.FutureLocationColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.CurrentLocationColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.LastLocationColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.BulkFileColumnFamily.NAME);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ while (iter.hasNext()) {
+ Key key = iter.next().getKey();
+
+ if (!key.getRow().equals(extent.getMetadataEntry())) {
+ throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
+ }
+
+ if (expectedColumnFamilies.contains(key.getColumnFamily())) {
+ continue;
+ }
+
+ if (expectedColumns.remove(new ColumnFQ(key))) {
+ continue;
+ }
+
+ throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
+ }
+ System.out.println("expectedColumns " + expectedColumns);
+ if (expectedColumns.size() > 1 || (expectedColumns.size() == 1)) {
+ throw new Exception("Not all expected columns seen " + extent + " " + expectedColumns);
+ }
+
+ SortedMap<FileRef,DataFileValue> fixedMapFiles = MetadataTableUtil.getDataFileSizes(extent, SystemCredentials.get());
+ verifySame(expectedMapFiles, fixedMapFiles);
+ }
+
+ private void verifySame(SortedMap<FileRef,DataFileValue> datafileSizes, SortedMap<FileRef,DataFileValue> fixedDatafileSizes) throws Exception {
+
+ if (!datafileSizes.keySet().containsAll(fixedDatafileSizes.keySet()) || !fixedDatafileSizes.keySet().containsAll(datafileSizes.keySet())) {
+ throw new Exception("Key sets not the same " + datafileSizes.keySet() + " != " + fixedDatafileSizes.keySet());
+ }
+
+ for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
+ DataFileValue dfv = entry.getValue();
+ DataFileValue otherDfv = fixedDatafileSizes.get(entry.getKey());
+
+ if (!dfv.equals(otherDfv)) {
+ throw new Exception(entry.getKey() + " dfv not equal " + dfv + " " + otherDfv);
+ }
+ }
+ }
+
+
+ public static void main(String[] args) throws Exception {
+ new SplitRecoveryIT().run();
+ }
+
+ @Test
+ public void test() throws Exception {
+ assertEquals(0, exec(SplitRecoveryIT.class).waitFor());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
index b74399d,0000000..fccc79f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
@@@ -1,101 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class TabletIT extends ConfigurableMacIT {
+
+ private static final int N = 1000;
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ Map<String,String> siteConfig = new HashMap<String,String>();
+ siteConfig.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "200");
+ siteConfig.put(Property.TSERV_MAXMEM.getKey(), "128M");
+ cfg.setDefaultMemory(256, MemoryUnit.MEGABYTE);
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void createTableTest() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ createTableTest(tableName, false);
+ createTableTest(tableName, true);
+ }
+
+ public void createTableTest(String tableName, boolean readOnly) throws Exception {
+ // create the test table within accumulo
+ Connector connector = getConnector();
+
+ if (!readOnly) {
+ TreeSet<Text> keys = new TreeSet<Text>();
+ for (int i = N / 100; i < N; i += N / 100) {
+ keys.add(new Text(String.format("%05d", i)));
+ }
+
+ // presplit
+ connector.tableOperations().create(tableName);
+ connector.tableOperations().addSplits(tableName, keys);
+ BatchWriter b = connector.createBatchWriter(tableName, new BatchWriterConfig());
+
+ // populate
+ for (int i = 0; i < N; i++) {
+ Mutation m = new Mutation(new Text(String.format("%05d", i)));
- m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
++ m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
+ b.addMutation(m);
+ }
+ b.close();
+ }
+
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ int count = 0;
+ for (Entry<Key,Value> elt : scanner) {
+ String expected = String.format("%05d", count);
+ assert (elt.getKey().getRow().toString().equals(expected));
+ count++;
+ }
+ assertEquals(N, count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
index 1dd0bf1,0000000..f2460cf
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@@ -1,303 -1,0 +1,304 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ByteArraySet;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class VisibilityIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = getUniqueNames(2);
+ String table = tableNames[0];
+ c.tableOperations().create(table);
+ String table2 = tableNames[1];
+ c.tableOperations().create(table2);
+ c.tableOperations().setProperty(table2, Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "DEFLABEL");
+
+ insertData(c, table);
+ queryData(c, table);
+ deleteData(c, table);
+
+ insertDefaultData(c, table2);
+ queryDefaultData(c, table2);
+
+ }
+
+ private static SortedSet<String> nss(String... labels) {
+ TreeSet<String> ts = new TreeSet<String>();
+
+ for (String s : labels) {
+ ts.add(s);
+ }
+
+ return ts;
+ }
+
+ private void mput(Mutation m, String cf, String cq, String cv, String val) {
- ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
- m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(Constants.UTF8)));
++ ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
++ m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(UTF_8)));
+ }
+
+ private void mputDelete(Mutation m, String cf, String cq, String cv) {
- ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
++ ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
+ m.putDelete(new Text(cf), new Text(cq), le);
+ }
+
+ private void insertData(Connector c, String tableName) throws Exception {
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m1 = new Mutation(new Text("row1"));
+
+ mput(m1, "cf1", "cq1", "", "v1");
+ mput(m1, "cf1", "cq1", "A", "v2");
+ mput(m1, "cf1", "cq1", "B", "v3");
+ mput(m1, "cf1", "cq1", "A&B", "v4");
+ mput(m1, "cf1", "cq1", "A&(L|M)", "v5");
+ mput(m1, "cf1", "cq1", "B&(L|M)", "v6");
+ mput(m1, "cf1", "cq1", "A&B&(L|M)", "v7");
+ mput(m1, "cf1", "cq1", "A&B&(L)", "v8");
+ mput(m1, "cf1", "cq1", "A&FOO", "v9");
+ mput(m1, "cf1", "cq1", "A&FOO&(L|M)", "v10");
+ mput(m1, "cf1", "cq1", "FOO", "v11");
+ mput(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)", "v12");
+ mput(m1, "cf1", "cq1", "A&B&(L|M|FOO)", "v13");
+
+ bw.addMutation(m1);
+ bw.close();
+ }
+
+ private void deleteData(Connector c, String tableName) throws Exception {
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m1 = new Mutation(new Text("row1"));
+
+ mputDelete(m1, "cf1", "cq1", "");
+ mputDelete(m1, "cf1", "cq1", "A");
+ mputDelete(m1, "cf1", "cq1", "A&B");
+ mputDelete(m1, "cf1", "cq1", "B&(L|M)");
+ mputDelete(m1, "cf1", "cq1", "A&B&(L)");
+ mputDelete(m1, "cf1", "cq1", "A&FOO&(L|M)");
+ mputDelete(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)");
+ mputDelete(m1, "cf1", "cq1", "FOO&A"); // should not delete anything
+
+ bw.addMutation(m1);
+ bw.close();
+
+ Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
+
+ expected.put(nss("A", "L"), nss("v5"));
+ expected.put(nss("A", "M"), nss("v5"));
+ expected.put(nss("B"), nss("v3"));
+ expected.put(nss("Z"), nss());
+ expected.put(nss("A", "B", "L"), nss("v7", "v13"));
+ expected.put(nss("A", "B", "M"), nss("v7", "v13"));
+ expected.put(nss("A", "B", "FOO"), nss("v13"));
+ expected.put(nss("FOO"), nss("v11"));
+ expected.put(nss("A", "FOO"), nss("v9"));
+
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
+ }
+
+ private void insertDefaultData(Connector c, String tableName) throws Exception {
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m1 = new Mutation(new Text("row1"));
+
+ mput(m1, "cf1", "cq1", "BASE", "v1");
+ mput(m1, "cf1", "cq2", "DEFLABEL", "v2");
+ mput(m1, "cf1", "cq3", "", "v3");
+
+ bw.addMutation(m1);
+ bw.close();
+ }
+
+ private static void uniqueCombos(List<Set<String>> all, Set<String> prefix, Set<String> suffix) {
+
+ all.add(prefix);
+
+ TreeSet<String> ss = new TreeSet<String>(suffix);
+
+ for (String s : suffix) {
+ TreeSet<String> ps = new TreeSet<String>(prefix);
+ ps.add(s);
+ ss.remove(s);
+
+ uniqueCombos(all, ps, ss);
+ }
+ }
+
+ private void queryData(Connector c, String tableName) throws Exception {
+ Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
+ expected.put(nss(), nss("v1"));
+ expected.put(nss("A"), nss("v2"));
+ expected.put(nss("A", "L"), nss("v5"));
+ expected.put(nss("A", "M"), nss("v5"));
+ expected.put(nss("B"), nss("v3"));
+ expected.put(nss("B", "L"), nss("v6"));
+ expected.put(nss("B", "M"), nss("v6"));
+ expected.put(nss("Z"), nss());
+ expected.put(nss("A", "B"), nss("v4"));
+ expected.put(nss("A", "B", "L"), nss("v7", "v8", "v13"));
+ expected.put(nss("A", "B", "M"), nss("v7", "v13"));
+ expected.put(nss("A", "B", "FOO"), nss("v13"));
+ expected.put(nss("FOO"), nss("v11"));
+ expected.put(nss("A", "FOO"), nss("v9"));
+ expected.put(nss("A", "FOO", "L"), nss("v10", "v12"));
+ expected.put(nss("A", "FOO", "M"), nss("v10", "v12"));
+ expected.put(nss("B", "FOO", "L"), nss("v12"));
+ expected.put(nss("B", "FOO", "M"), nss("v12"));
+
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "L", "M", "Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected);
+ }
+
+ private void queryData(Connector c, String tableName, Set<String> allAuths, Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws Exception {
+
+ c.securityOperations().changeUserAuthorizations("root", new Authorizations(nbas(userAuths)));
+
+ ArrayList<Set<String>> combos = new ArrayList<Set<String>>();
+ uniqueCombos(combos, nss(), allAuths);
+
+ for (Set<String> set1 : combos) {
+ Set<String> e = new TreeSet<String>();
+ for (Set<String> set2 : combos) {
+
+ set2 = new HashSet<String>(set2);
+ set2.retainAll(userAuths);
+
+ if (set1.containsAll(set2) && expected.containsKey(set2)) {
+ e.addAll(expected.get(set2));
+ }
+ }
+
+ set1.retainAll(userAuths);
+ verify(c, tableName, set1, e);
+ }
+
+ }
+
+ private void queryDefaultData(Connector c, String tableName) throws Exception {
+ Scanner scanner;
+
+ // should return no records
+ c.securityOperations().changeUserAuthorizations("root", new Authorizations("BASE", "DEFLABEL"));
+ scanner = getConnector().createScanner(tableName, new Authorizations());
+ verifyDefault(scanner, 0);
+
+ // should return one record
+ scanner = getConnector().createScanner(tableName, new Authorizations("BASE"));
+ verifyDefault(scanner, 1);
+
+ // should return all three records
+ scanner = getConnector().createScanner(tableName, new Authorizations("BASE", "DEFLABEL"));
+ verifyDefault(scanner, 3);
+ }
+
+ private void verifyDefault(Scanner scanner, int expectedCount) throws Exception {
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner)
+ --expectedCount;
+ if (expectedCount != 0)
+ throw new Exception(" expected count !=0 " + expectedCount);
+ }
+
+ private void verify(Connector c, String tableName, Set<String> auths, Set<String> expectedValues) throws Exception {
+ ByteArraySet bas = nbas(auths);
+
+ try {
+ verify(c, tableName, bas, expectedValues.toArray(new String[0]));
+ } catch (Exception e) {
+ throw new Exception("Verification failed auths=" + auths + " exp=" + expectedValues, e);
+ }
+ }
+
+ private ByteArraySet nbas(Set<String> auths) {
+ ByteArraySet bas = new ByteArraySet();
+ for (String auth : auths) {
- bas.add(auth.getBytes(Constants.UTF8));
++ bas.add(auth.getBytes(UTF_8));
+ }
+ return bas;
+ }
+
+ private void verify(Connector c, String tableName, ByteArraySet nss, String... expected) throws Exception {
+ Scanner scanner = c.createScanner(tableName, new Authorizations(nss));
+ verify(scanner.iterator(), expected);
+
+ BatchScanner bs = getConnector().createBatchScanner(tableName, new Authorizations(nss), 3);
+ bs.setRanges(Collections.singleton(new Range()));
+ verify(bs.iterator(), expected);
+ bs.close();
+ }
+
+ private void verify(Iterator<Entry<Key,Value>> iter, String... expected) throws Exception {
+ HashSet<String> valuesSeen = new HashSet<String>();
+
+ while (iter.hasNext()) {
+ Entry<Key,Value> entry = iter.next();
+ if (valuesSeen.contains(entry.getValue().toString())) {
+ throw new Exception("Value seen twice");
+ }
+ valuesSeen.add(entry.getValue().toString());
+ }
+
+ for (String ev : expected) {
+ if (!valuesSeen.remove(ev)) {
+ throw new Exception("Did not see expected value " + ev);
+ }
+ }
+
+ if (valuesSeen.size() != 0) {
+ throw new Exception("Saw more values than expected " + valuesSeen);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
[35/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
index d0e6aea,0000000..258080c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
@@@ -1,775 -1,0 +1,777 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.SecurityOperationsImpl;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.master.thrift.FateOperation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.server.security.handler.Authorizor;
+import org.apache.accumulo.server.security.handler.PermissionHandler;
+import org.apache.accumulo.server.security.handler.ZKAuthenticator;
+import org.apache.accumulo.server.security.handler.ZKAuthorizor;
+import org.apache.accumulo.server.security.handler.ZKPermHandler;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * Utility class for performing various security operations with the appropriate checks
+ */
+public class SecurityOperation {
+ private static final Logger log = Logger.getLogger(SecurityOperationsImpl.class);
+
+ protected Authorizor authorizor;
+ protected Authenticator authenticator;
+ protected PermissionHandler permHandle;
+ private static String rootUserName = null;
+ private final ZooCache zooCache;
+ private final String ZKUserPath;
+
+ static SecurityOperation instance;
+
+ public static synchronized SecurityOperation getInstance() {
+ String instanceId = HdfsZooInstance.getInstance().getInstanceID();
+ return getInstance(instanceId, false);
+ }
+
+ public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) {
+ if (instance == null) {
+ instance = new SecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, initialize),
+ instanceId);
+ }
+ return instance;
+ }
+
+ protected static Authorizor getAuthorizor(String instanceId, boolean initialize) {
+ Authorizor toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHORIZOR, Authorizor.class,
+ ZKAuthorizor.getInstance());
+ toRet.initialize(instanceId, initialize);
+ return toRet;
+ }
+
+ protected static Authenticator getAuthenticator(String instanceId, boolean initialize) {
+ Authenticator toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHENTICATOR, Authenticator.class,
+ ZKAuthenticator.getInstance());
+ toRet.initialize(instanceId, initialize);
+ return toRet;
+ }
+
+ protected static PermissionHandler getPermHandler(String instanceId, boolean initialize) {
+ PermissionHandler toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_PERMISSION_HANDLER,
+ PermissionHandler.class, ZKPermHandler.getInstance());
+ toRet.initialize(instanceId, initialize);
+ return toRet;
+ }
+
+ protected SecurityOperation(String instanceId) {
+ ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
+ zooCache = new ZooCache();
+ }
+
+ public SecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) {
+ this(instanceId);
+ authorizor = author;
+ authenticator = authent;
+ permHandle = pm;
+
+ if (!authorizor.validSecurityHandlers(authenticator, pm) || !authenticator.validSecurityHandlers(authorizor, pm)
+ || !permHandle.validSecurityHandlers(authent, author))
+ throw new RuntimeException(authorizor + ", " + authenticator + ", and " + pm
+ + " do not play nice with eachother. Please choose authentication and authorization mechanisms that are compatible with one another.");
+ }
+
+ public void initializeSecurity(TCredentials credentials, String rootPrincipal, byte[] token) throws AccumuloSecurityException, ThriftSecurityException {
+ authenticate(credentials);
+
+ if (!isSystemUser(credentials))
+ throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ authenticator.initializeSecurity(credentials, rootPrincipal, token);
+ authorizor.initializeSecurity(credentials, rootPrincipal);
+ permHandle.initializeSecurity(credentials, rootPrincipal);
+ try {
+ permHandle.grantTablePermission(rootPrincipal, MetadataTable.ID, TablePermission.ALTER_TABLE);
+ } catch (TableNotFoundException e) {
+ // Shouldn't happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ public synchronized String getRootUsername() {
+ if (rootUserName == null)
- rootUserName = new String(zooCache.get(ZKUserPath), Constants.UTF8);
++ rootUserName = new String(zooCache.get(ZKUserPath), UTF_8);
+ return rootUserName;
+ }
+
+ public boolean isSystemUser(TCredentials credentials) {
+ return SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName());
+ }
+
+ protected void authenticate(TCredentials credentials) throws ThriftSecurityException {
+ if (!credentials.getInstanceId().equals(HdfsZooInstance.getInstance().getInstanceID()))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.INVALID_INSTANCEID);
+
+ Credentials creds = Credentials.fromThrift(credentials);
+ if (isSystemUser(credentials)) {
+ if (!(SystemCredentials.get().equals(creds))) {
+ throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
+ }
+ } else {
+ try {
+ if (!authenticator.authenticateUser(creds.getPrincipal(), creds.getToken())) {
+ throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
+ }
+ } catch (AccumuloSecurityException e) {
+ log.debug(e);
+ throw e.asThriftException();
+ }
+ }
+ }
+
+ public boolean canAskAboutUser(TCredentials credentials, String user) throws ThriftSecurityException {
+ // Authentication done in canPerformSystemActions
+ if (!(canPerformSystemActions(credentials) || credentials.getPrincipal().equals(user)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return true;
+ }
+
+ public boolean authenticateUser(TCredentials credentials, TCredentials toAuth) throws ThriftSecurityException {
+ canAskAboutUser(credentials, toAuth.getPrincipal());
+ // User is already authenticated from canAskAboutUser
+ if (credentials.equals(toAuth))
+ return true;
+ try {
+ Credentials toCreds = Credentials.fromThrift(toAuth);
+ return authenticator.authenticateUser(toCreds.getPrincipal(), toCreds.getToken());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public Authorizations getUserAuthorizations(TCredentials credentials, String user) throws ThriftSecurityException {
+ authenticate(credentials);
+
+ targetUserExists(user);
+
+ if (!credentials.getPrincipal().equals(user) && !hasSystemPermission(credentials, SystemPermission.SYSTEM, false))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ try {
+ return authorizor.getCachedUserAuthorizations(user);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public Authorizations getUserAuthorizations(TCredentials credentials) throws ThriftSecurityException {
+ // system user doesn't need record-level authorizations for the tables it reads
+ if (isSystemUser(credentials)) {
+ authenticate(credentials);
+ return Authorizations.EMPTY;
+ }
+ return getUserAuthorizations(credentials, credentials.getPrincipal());
+ }
+
+ public boolean userHasAuthorizations(TCredentials credentials, List<ByteBuffer> list) throws ThriftSecurityException {
+ authenticate(credentials);
+
+ if (isSystemUser(credentials)) {
+ // system user doesn't need record-level authorizations for the tables it reads (for now)
+ return list.isEmpty();
+ }
+
+ try {
+ return authorizor.isValidAuthorizations(credentials.getPrincipal(), list);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ private boolean hasSystemPermission(TCredentials credentials, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
+ return hasSystemPermissionWithNamespaceId(credentials, permission, null, useCached);
+ }
+
+ /**
+ * Checks if a user has a system permission
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ private boolean hasSystemPermissionWithNamespaceId(TCredentials credentials, SystemPermission permission, String namespaceId, boolean useCached)
+ throws ThriftSecurityException {
+ if (isSystemUser(credentials))
+ return true;
+
+ if (_hasSystemPermission(credentials.getPrincipal(), permission, useCached))
+ return true;
+ if (namespaceId != null) {
+ return _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
+ }
+
+ return false;
+ }
+
+ /**
+ * Checks if a user has a system permission<br/>
+ * This cannot check if a system user has permission.
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ private boolean _hasSystemPermission(String user, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
+ if (user.equals(getRootUsername()))
+ return true;
+
+ targetUserExists(user);
+
+ try {
+ if (useCached)
+ return permHandle.hasCachedSystemPermission(user, permission);
+ return permHandle.hasSystemPermission(user, permission);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ /**
+ * Checks if a user has a table permission
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ protected boolean hasTablePermission(TCredentials credentials, String tableId, String namespaceId, TablePermission permission, boolean useCached) throws ThriftSecurityException {
+ if (isSystemUser(credentials))
+ return true;
+ return _hasTablePermission(credentials.getPrincipal(), tableId, permission, useCached)
+ || _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
+ }
+
+ /**
+ * Checks if a user has a table permission<br/>
+ * This cannot check if a system user has permission.
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ protected boolean _hasTablePermission(String user, String table, TablePermission permission, boolean useCached) throws ThriftSecurityException {
+ targetUserExists(user);
+
+ if ((table.equals(MetadataTable.ID) || table.equals(RootTable.ID)) && permission.equals(TablePermission.READ))
+ return true;
+
+ try {
+ if (useCached)
+ return permHandle.hasCachedTablePermission(user, table, permission);
+ return permHandle.hasTablePermission(user, table, permission);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(user, SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ /**
+ * Checks if a user has a namespace permission<br/>
+ * This cannot check if a system user has permission.
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ protected boolean _hasNamespacePermission(String user, String namespace, NamespacePermission permission, boolean useCached) throws ThriftSecurityException {
+ if (permission == null)
+ return false;
+
+ targetUserExists(user);
+
+ if (namespace.equals(Namespaces.ACCUMULO_NAMESPACE_ID) && permission.equals(NamespacePermission.READ))
+ return true;
+
+ try {
+ if (useCached)
+ return permHandle.hasCachedNamespacePermission(user, namespace, permission);
+ return permHandle.hasNamespacePermission(user, namespace, permission);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(user, SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ // some people just aren't allowed to ask about other users; here are those who can ask
+ private boolean canAskAboutOtherUsers(TCredentials credentials, String user) throws ThriftSecurityException {
+ authenticate(credentials);
+ return credentials.getPrincipal().equals(user) || hasSystemPermission(credentials, SystemPermission.SYSTEM, false)
+ || hasSystemPermission(credentials, SystemPermission.CREATE_USER, false) || hasSystemPermission(credentials, SystemPermission.ALTER_USER, false)
+ || hasSystemPermission(credentials, SystemPermission.DROP_USER, false);
+ }
+
+ private void targetUserExists(String user) throws ThriftSecurityException {
+ if (user.equals(getRootUsername()))
+ return;
+ try {
+ if (!authenticator.userExists(user))
+ throw new ThriftSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public boolean canScan(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, true);
+ }
+
+ public boolean canScan(TCredentials credentials, String tableId, String namespaceId, TRange range, List<TColumn> columns, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
+ return canScan(credentials, tableId, namespaceId);
+ }
+
+ public boolean canScan(TCredentials credentials, String table, String namespaceId, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
+ return canScan(credentials, table, namespaceId);
+ }
+
+ public boolean canWrite(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasTablePermission(credentials, tableId, namespaceId, TablePermission.WRITE, true);
+ }
+
+ public boolean canConditionallyUpdate(TCredentials credentials, String tableID, String namespaceId, List<ByteBuffer> authorizations) throws ThriftSecurityException {
+
+ authenticate(credentials);
+
+ return hasTablePermission(credentials, tableID, namespaceId, TablePermission.WRITE, true) && hasTablePermission(credentials, tableID, namespaceId, TablePermission.READ, true);
+ }
+
+ public boolean canSplitTablet(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasSystemPermissionWithNamespaceId(credentials, SystemPermission.SYSTEM, namespaceId, false)
+ || hasTablePermission(credentials, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ /**
+ * This is the check to perform any system action. This includes tserver's loading of a tablet, shutting the system down, or altering system properties.
+ */
+ public boolean canPerformSystemActions(TCredentials credentials) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermission(credentials, SystemPermission.SYSTEM, false);
+ }
+
+ public boolean canFlush(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canAlterTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false)
+ || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false);
+ }
+
+ public boolean canCreateTable(TCredentials c, String table, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, namespaceId, false);
+ }
+
+ public boolean canRenameTable(TCredentials c, String tableId, String oldTableName, String newTableName, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canCloneTable(TCredentials c, String tableId, String tableName, String destinationNamespaceId, String srcNamespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, destinationNamespaceId, false) && hasTablePermission(c, tableId, srcNamespaceId, TablePermission.READ, false);
+ }
+
+ public boolean canDeleteTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.DROP_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.DROP_TABLE, false);
+ }
+
+ public boolean canOnlineOfflineTable(TCredentials c, String tableId, FateOperation op, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
+ || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canMerge(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
+ || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canDeleteRange(TCredentials c, String tableId, String tableName, Text startRow, Text endRow, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
+ }
+
+ public boolean canBulkImport(TCredentials c, String tableId, String tableName, String dir, String failDir, String namespaceId) throws ThriftSecurityException {
+ return canBulkImport(c, tableId, namespaceId);
+ }
+
+ public boolean canBulkImport(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasTablePermission(c, tableId, namespaceId, TablePermission.BULK_IMPORT, false);
+ }
+
+ public boolean canCompact(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
+ }
+
+ public boolean canChangeAuthorizations(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermission(c, SystemPermission.ALTER_USER, false);
+ }
+
+ public boolean canChangePassword(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ return c.getPrincipal().equals(user) || hasSystemPermission(c, SystemPermission.ALTER_USER, false);
+ }
+
+ public boolean canCreateUser(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermission(c, SystemPermission.CREATE_USER, false);
+ }
+
+ public boolean canDropUser(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ if (user.equals(getRootUsername()))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return hasSystemPermission(c, SystemPermission.DROP_USER, false);
+ }
+
+ public boolean canGrantSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
+ authenticate(c);
+ // can't grant GRANT
+ if (sysPerm.equals(SystemPermission.GRANT))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
+ return hasSystemPermission(c, SystemPermission.GRANT, false);
+ }
+
+ public boolean canGrantTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
+ }
+
+ public boolean canGrantNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
+ return canModifyNamespacePermission(c, user, namespace);
+ }
+
+ private boolean canModifyNamespacePermission(TCredentials c, String user, String namespace) throws ThriftSecurityException {
+ authenticate(c);
+ // The one case where Table/SystemPermission -> NamespacePermission breaks down. The alternative is to make SystemPermission.ALTER_NAMESPACE provide
+ // NamespacePermission.GRANT & ALTER_NAMESPACE, but then it would cause some permission checks to succeed with GRANT when they shouldn't
+
+ // This is a bit hackier then I (vines) wanted, but I think this one hackiness makes the overall SecurityOperations more succinct.
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_NAMESPACE, namespace, false)
+ || hasNamespacePermission(c, c.principal, namespace, NamespacePermission.GRANT);
+ }
+
+ public boolean canRevokeSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
+ authenticate(c);
+ // can't modify root user
+ if (user.equals(getRootUsername()))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ // can't revoke GRANT
+ if (sysPerm.equals(SystemPermission.GRANT))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
+
+ return hasSystemPermission(c, SystemPermission.GRANT, false);
+ }
+
+ public boolean canRevokeTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
+ }
+
+ public boolean canRevokeNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
+ return canModifyNamespacePermission(c, user, namespace);
+ }
+
+ public void changeAuthorizations(TCredentials credentials, String user, Authorizations authorizations) throws ThriftSecurityException {
+ if (!canChangeAuthorizations(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ authorizor.changeAuthorizations(user, authorizations);
+ log.info("Changed authorizations for user " + user + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException ase) {
+ throw ase.asThriftException();
+ }
+ }
+
+ public void changePassword(TCredentials credentials, Credentials toChange) throws ThriftSecurityException {
+ if (!canChangePassword(credentials, toChange.getPrincipal()))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ AuthenticationToken token = toChange.getToken();
+ authenticator.changePassword(toChange.getPrincipal(), token);
+ log.info("Changed password for user " + toChange.getPrincipal() + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void createUser(TCredentials credentials, Credentials newUser, Authorizations authorizations) throws ThriftSecurityException {
+ if (!canCreateUser(credentials, newUser.getPrincipal()))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ AuthenticationToken token = newUser.getToken();
+ authenticator.createUser(newUser.getPrincipal(), token);
+ authorizor.initUser(newUser.getPrincipal());
+ permHandle.initUser(newUser.getPrincipal());
+ log.info("Created user " + newUser.getPrincipal() + " at the request of user " + credentials.getPrincipal());
+ if (canChangeAuthorizations(credentials, newUser.getPrincipal()))
+ authorizor.changeAuthorizations(newUser.getPrincipal(), authorizations);
+ } catch (AccumuloSecurityException ase) {
+ throw ase.asThriftException();
+ }
+ }
+
+ public void dropUser(TCredentials credentials, String user) throws ThriftSecurityException {
+ if (!canDropUser(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ authorizor.dropUser(user);
+ authenticator.dropUser(user);
+ permHandle.cleanUser(user);
+ log.info("Deleted user " + user + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void grantSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
+ if (!canGrantSystem(credentials, user, permissionById))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.grantSystemPermission(user, permissionById);
+ log.info("Granted system permission " + permissionById + " for user " + user + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void grantTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
+ if (!canGrantTable(c, user, tableId, namespaceId))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.grantTablePermission(user, tableId, permission);
+ log.info("Granted table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ public void grantNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
+ if (!canGrantNamespace(c, user, namespace))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.grantNamespacePermission(user, namespace, permission);
+ log.info("Granted namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
+ + c.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ public void revokeSystemPermission(TCredentials credentials, String user, SystemPermission permission) throws ThriftSecurityException {
+ if (!canRevokeSystem(credentials, user, permission))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.revokeSystemPermission(user, permission);
+ log.info("Revoked system permission " + permission + " for user " + user + " at the request of user " + credentials.getPrincipal());
+
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void revokeTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
+ if (!canRevokeTable(c, user, tableId, namespaceId))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.revokeTablePermission(user, tableId, permission);
+ log.info("Revoked table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
+
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ public void revokeNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
+ if (!canRevokeNamespace(c, user, namespace))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.revokeNamespacePermission(user, namespace, permission);
+ log.info("Revoked namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
+ + c.getPrincipal());
+
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ public boolean hasSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
+ if (!canAskAboutOtherUsers(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return _hasSystemPermission(user, permissionById, false);
+ }
+
+ public boolean hasTablePermission(TCredentials credentials, String user, String tableId, TablePermission permissionById) throws ThriftSecurityException {
+ if (!canAskAboutOtherUsers(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return _hasTablePermission(user, tableId, permissionById, false);
+ }
+
+ public boolean hasNamespacePermission(TCredentials credentials, String user, String namespace, NamespacePermission permissionById)
+ throws ThriftSecurityException {
+ if (!canAskAboutOtherUsers(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return _hasNamespacePermission(user, namespace, permissionById, false);
+ }
+
+ public Set<String> listUsers(TCredentials credentials) throws ThriftSecurityException {
+ authenticate(credentials);
+ try {
+ return authenticator.listUsers();
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void deleteTable(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ if (!canDeleteTable(credentials, tableId, namespaceId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ permHandle.cleanTablePermissions(tableId);
+ } catch (AccumuloSecurityException e) {
+ e.setUser(credentials.getPrincipal());
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ public void deleteNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
+ if (!canDeleteNamespace(credentials, namespace))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ permHandle.cleanNamespacePermissions(namespace);
+ } catch (AccumuloSecurityException e) {
+ e.setUser(credentials.getPrincipal());
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ public boolean canExport(TCredentials credentials, String tableId, String tableName, String exportDir, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, false);
+ }
+
+ public boolean canImport(TCredentials credentials, String tableName, String importDir, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.CREATE_TABLE, namespaceId, false);
+ }
+
+ public boolean canAlterNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
+ }
+
+ public boolean canCreateNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
+ return canCreateNamespace(credentials);
+ }
+
+ private boolean canCreateNamespace(TCredentials credentials) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermission(credentials, SystemPermission.CREATE_NAMESPACE, false);
+ }
+
+ public boolean canDeleteNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.DROP_NAMESPACE, namespaceId, false);
+ }
+
+ public boolean canRenameNamespace(TCredentials credentials, String namespaceId, String oldName, String newName) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 19e7ff7,0000000..872fe11
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@@ -1,149 -1,0 +1,151 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecurityPermission;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Credentials for the system services.
+ *
+ * @since 1.6.0
+ */
+public final class SystemCredentials extends Credentials {
+
+ private static final SecurityPermission SYSTEM_CREDENTIALS_PERMISSION = new SecurityPermission("systemCredentialsPermission");
+
+ private static SystemCredentials SYSTEM_CREDS = null;
+ private static final String SYSTEM_PRINCIPAL = "!SYSTEM";
+
+ private final TCredentials AS_THRIFT;
+
+ SystemCredentials(Instance instance) {
+ super(SYSTEM_PRINCIPAL, SystemToken.get(instance));
+ AS_THRIFT = super.toThrift(instance);
+ }
+
+ public static SystemCredentials get() {
+ check_permission();
+ if (SYSTEM_CREDS == null) {
+ SYSTEM_CREDS = new SystemCredentials(HdfsZooInstance.getInstance());
+ }
+ return SYSTEM_CREDS;
+ }
+
+ private static void check_permission() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(SYSTEM_CREDENTIALS_PERMISSION);
+ }
+ }
+
+ public static SystemCredentials get(Instance instance) {
+ check_permission();
+ /* Special case to avoid duplicating SYSTEM_CREDS */
+ if (null != SYSTEM_CREDS) {
+ if (SYSTEM_CREDS.AS_THRIFT.getInstanceId().equals(instance.getInstanceID())) {
+ return SYSTEM_CREDS;
+ }
+ }
+ return new SystemCredentials(instance);
+ }
+
+ @Override
+ public TCredentials toThrift(Instance instance) {
+ if (!AS_THRIFT.getInstanceId().equals(instance.getInstanceID()))
+ throw new IllegalArgumentException("Unexpected instance used for " + SystemCredentials.class.getSimpleName() + ": " + instance.getInstanceID());
+ return AS_THRIFT;
+ }
+
+ /**
+ * An {@link AuthenticationToken} type for Accumulo servers for inter-server communication.
+ *
+ * @since 1.6.0
+ */
+ public static final class SystemToken extends PasswordToken {
+
+ /**
+ * A Constructor for {@link Writable}.
+ */
+ public SystemToken() {}
+
+ private SystemToken(byte[] systemPassword) {
+ super(systemPassword);
+ }
+
+ private static SystemToken get(Instance instance) {
- byte[] instanceIdBytes = instance.getInstanceID().getBytes(Constants.UTF8);
++ byte[] instanceIdBytes = instance.getInstanceID().getBytes(UTF_8);
+ byte[] confChecksum;
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance(Constants.PW_HASH_ALGORITHM);
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("Failed to compute configuration checksum", e);
+ }
+
+ // seed the config with the version and instance id, so at least it's not empty
- md.update(ServerConstants.WIRE_VERSION.toString().getBytes(Constants.UTF8));
++ md.update(ServerConstants.WIRE_VERSION.toString().getBytes(UTF_8));
+ md.update(instanceIdBytes);
+
+ for (Entry<String,String> entry : ServerConfiguration.getSiteConfiguration()) {
+ // only include instance properties
+ if (entry.getKey().startsWith(Property.INSTANCE_PREFIX.toString())) {
- md.update(entry.getKey().getBytes(Constants.UTF8));
- md.update(entry.getValue().getBytes(Constants.UTF8));
++ md.update(entry.getKey().getBytes(UTF_8));
++ md.update(entry.getValue().getBytes(UTF_8));
+ }
+ }
+ confChecksum = md.digest();
+
+ int wireVersion = ServerConstants.WIRE_VERSION;
+
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(3 * (Integer.SIZE / Byte.SIZE) + instanceIdBytes.length + confChecksum.length);
+ DataOutputStream out = new DataOutputStream(bytes);
+ try {
+ out.write(wireVersion * -1);
+ out.write(instanceIdBytes.length);
+ out.write(instanceIdBytes);
+ out.write(confChecksum.length);
+ out.write(confChecksum);
+ } catch (IOException e) {
+ // this is impossible with ByteArrayOutputStream; crash hard if this happens
+ throw new RuntimeException(e);
+ }
+ return new SystemToken(Base64.encodeBase64(bytes.toByteArray()));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
index 5f6ff71,0000000..fa1632c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
@@@ -1,214 -1,0 +1,216 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+// Utility class for adding all authentication info into ZK
+public final class ZKAuthenticator implements Authenticator {
+ static final Logger log = Logger.getLogger(ZKAuthenticator.class);
+ private static Authenticator zkAuthenticatorInstance = null;
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authenticator getInstance() {
+ if (zkAuthenticatorInstance == null)
+ zkAuthenticatorInstance = new ZKAuthenticator();
+ return zkAuthenticatorInstance;
+ }
+
+ public ZKAuthenticator() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException {
+ try {
+ // remove old settings from zookeeper first, if any
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ synchronized (zooCache) {
+ zooCache.clear();
+ if (zoo.exists(ZKUserPath)) {
+ zoo.recursiveDelete(ZKUserPath, NodeMissingPolicy.SKIP);
+ log.info("Removed " + ZKUserPath + "/" + " from zookeeper");
+ }
+
+ // prep parent node of users with root username
- zoo.putPersistentData(ZKUserPath, principal.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(ZKUserPath, principal.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ constructUser(principal, ZKSecurityTool.createPass(token));
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up the user in ZK for the provided user. No checking for existence is done here, it should be done before calling.
+ */
+ private void constructUser(String user, byte[] pass) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPrivatePersistentData(ZKUserPath + "/" + user, pass, NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public Set<String> listUsers() {
+ return new TreeSet<String>(zooCache.getChildren(ZKUserPath));
+ }
+
+ /**
+ * Creates a user with no permissions whatsoever
+ */
+ @Override
+ public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ try {
+ if (!(token instanceof PasswordToken))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
+ PasswordToken pt = (PasswordToken) token;
+ constructUser(principal, ZKSecurityTool.createPass(pt.getPassword()));
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_EXISTS, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ }
+ }
+
+ @Override
+ public void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ if (!(token instanceof PasswordToken))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
+ PasswordToken pt = (PasswordToken) token;
+ if (userExists(principal)) {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + principal);
+ ZooReaderWriter.getInstance().putPrivatePersistentData(ZKUserPath + "/" + principal, ZKSecurityTool.createPass(pt.getPassword()),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
+ }
+ } else
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
+ }
+
+ /**
+ * Checks if a user exists
+ */
+ @Override
+ public boolean userExists(String user) {
+ return zooCache.get(ZKUserPath + "/" + user) != null;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ if (!(token instanceof PasswordToken))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
+ PasswordToken pt = (PasswordToken) token;
+ byte[] pass;
+ String zpath = ZKUserPath + "/" + principal;
+ pass = zooCache.get(zpath);
+ boolean result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
+ if (!result) {
+ zooCache.clear(zpath);
+ pass = zooCache.get(zpath);
+ result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
+ }
+ return result;
+ }
+
+ @Override
+ public Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes() {
+ Set<Class<? extends AuthenticationToken>> cs = new HashSet<Class<? extends AuthenticationToken>>();
+ cs.add(PasswordToken.class);
+ return cs;
+ }
+
+ @Override
+ public boolean validTokenClass(String tokenClass) {
+ return tokenClass.equals(PasswordToken.class.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 75b73fc,0000000..78597ad
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,171 -1,0 +1,172 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
+ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Authorizations.EMPTY;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the metadata tables
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
- zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
+ Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
+ for (ByteBuffer auth : auths)
+ if (!userauths.contains(auth))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index be4806b,0000000..41f880d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,515 -1,0 +1,516 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private String ZKNamespacePath;
+ private final ZooCache zooCache;
+ private final String ZKUserSysPerms = "/System";
+ private final String ZKUserTablePerms = "/Tables";
+ private final String ZKUserNamespacePerms = "/Namespaces";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ zrw.sync(path);
+ serializedPerms = zrw.getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ zrw.getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
+ byte[] serializedPerms;
+ final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
+ zrw.sync(path);
+ serializedPerms = zrw.getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the namespace was just deleted?
+ try {
+ // check for existence:
+ zrw.getData(ZKNamespacePath + "/" + namespace, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ Set<NamespacePermission> namespacePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null)
+ namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ else
+ namespacePerms = new TreeSet<NamespacePermission>();
+
+ try {
+ if (namespacePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+
+ // User had no namespace permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ try {
+ if (namespacePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (namespacePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the system tables
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
+ Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
- zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
+ createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ /**
+ * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getInstance().sync(path);
+ perms = ZooReaderWriter.getInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
[15/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index e8d5bbf,0000000..002389a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@@ -1,399 -1,0 +1,400 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.KeeperException.NotEmptyException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import com.google.common.net.HostAndPort;
+
+public class LiveTServerSet implements Watcher {
+
+ public interface Listener {
+ void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added);
+ }
+
+ private static final Logger log = Logger.getLogger(LiveTServerSet.class);
+
+ private final Listener cback;
+ private final Instance instance;
+ private final AccumuloConfiguration conf;
+ private ZooCache zooCache;
+
+ public class TServerConnection {
+ private final HostAndPort address;
+
+ public TServerConnection(HostAndPort addr) throws TException {
+ address = addr;
+ }
+
+ private String lockString(ZooLock mlock) {
+ return mlock.getLockID().serialize(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK);
+ }
+
+ public void assignTablet(ZooLock lock, KeyExtent extent) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.loadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void unloadTablet(ZooLock lock, KeyExtent extent, boolean save) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.unloadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift(), save);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public TabletServerStatus getTableMap(boolean usePooledConnection) throws TException, ThriftSecurityException {
+
+ if (usePooledConnection == true)
+ throw new UnsupportedOperationException();
+
+ TTransport transport = ThriftUtil.createTransport(address, conf);
+
+ try {
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ return client.getTabletServerStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
+ } finally {
+ if (transport != null)
+ transport.close();
+ }
+ }
+
+ public void halt(ZooLock lock) throws TException, ThriftSecurityException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.halt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void fastHalt(ZooLock lock) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.fastHalt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void flush(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.flush(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
+ startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void chop(ZooLock lock, KeyExtent extent) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.chop(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void splitTablet(ZooLock lock, KeyExtent extent, Text splitPoint) throws TException, ThriftSecurityException, NotServingTabletException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.splitTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), extent.toThrift(),
+ ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength()));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void flushTablet(ZooLock lock, KeyExtent extent) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.flushTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void compact(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.compact(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
+ startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public boolean isActive(long tid) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ return client.isActive(Tracer.traceInfo(), tid);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ }
+
+ static class TServerInfo {
+ TServerConnection connection;
+ TServerInstance instance;
+
+ TServerInfo(TServerInstance instance, TServerConnection connection) {
+ this.connection = connection;
+ this.instance = instance;
+ }
+ };
+
+ // The set of active tservers with locks, indexed by their name in zookeeper
+ private Map<String,TServerInfo> current = new HashMap<String,TServerInfo>();
+ // as above, indexed by TServerInstance
+ private Map<TServerInstance,TServerInfo> currentInstances = new HashMap<TServerInstance,TServerInfo>();
+
+ // The set of entries in zookeeper without locks, and the first time each was noticed
+ private Map<String,Long> locklessServers = new HashMap<String,Long>();
+
+ public LiveTServerSet(Instance instance, AccumuloConfiguration conf, Listener cback) {
+ this.cback = cback;
+ this.instance = instance;
+ this.conf = conf;
+
+ }
+
+ public synchronized ZooCache getZooCache() {
+ if (zooCache == null)
+ zooCache = new ZooCache(this);
+ return zooCache;
+ }
+
+ public synchronized void startListeningForTabletServerChanges() {
+ scanServers();
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ scanServers();
+ }
+ }, 0, 5000);
+ }
+
+ public synchronized void scanServers() {
+ try {
+ final Set<TServerInstance> updates = new HashSet<TServerInstance>();
+ final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
+
+ final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+
+ HashSet<String> all = new HashSet<String>(current.keySet());
+ all.addAll(getZooCache().getChildren(path));
+
+ locklessServers.keySet().retainAll(all);
+
+ for (String zPath : all) {
+ checkServer(updates, doomed, path, zPath);
+ }
+
+ // log.debug("Current: " + current.keySet());
+ if (!doomed.isEmpty() || !updates.isEmpty())
+ this.cback.update(this, doomed, updates);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private void deleteServerNode(String serverNode) throws InterruptedException, KeeperException {
+ try {
+ ZooReaderWriter.getInstance().delete(serverNode, -1);
+ } catch (NotEmptyException ex) {
+ // race condition: tserver created the lock after our last check; we'll see it at the next check
+ } catch (NoNodeException nne) {
+ // someone else deleted it
+ }
+ }
+
+ private synchronized void checkServer(final Set<TServerInstance> updates, final Set<TServerInstance> doomed, final String path, final String zPath)
+ throws TException, InterruptedException, KeeperException {
+
+ TServerInfo info = current.get(zPath);
+
+ final String lockPath = path + "/" + zPath;
+ Stat stat = new Stat();
+ byte[] lockData = ZooLock.getLockData(getZooCache(), lockPath, stat);
+
+ if (lockData == null) {
+ if (info != null) {
+ doomed.add(info.instance);
+ current.remove(zPath);
+ currentInstances.remove(info.instance);
+ }
+
+ Long firstSeen = locklessServers.get(zPath);
+ if (firstSeen == null) {
+ locklessServers.put(zPath, System.currentTimeMillis());
+ } else if (System.currentTimeMillis() - firstSeen > 10 * 60 * 1000) {
+ deleteServerNode(path + "/" + zPath);
+ locklessServers.remove(zPath);
+ }
+ } else {
+ locklessServers.remove(zPath);
- ServerServices services = new ServerServices(new String(lockData, Constants.UTF8));
++ ServerServices services = new ServerServices(new String(lockData, UTF_8));
+ HostAndPort client = services.getAddress(ServerServices.Service.TSERV_CLIENT);
+ TServerInstance instance = new TServerInstance(client, stat.getEphemeralOwner());
+
+ if (info == null) {
+ updates.add(instance);
+ TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
+ current.put(zPath, tServerInfo);
+ currentInstances.put(instance, tServerInfo);
+ } else if (!info.instance.equals(instance)) {
+ doomed.add(info.instance);
+ updates.add(instance);
+ TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
+ current.put(zPath, tServerInfo);
+ currentInstances.put(info.instance, tServerInfo);
+ }
+ }
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+
+ // its important that these event are propagated by ZooCache, because this ensures when reading zoocache that is has already processed the event and cleared
+ // relevant nodes before code below reads from zoocache
+
+ if (event.getPath() != null) {
+ if (event.getPath().endsWith(Constants.ZTSERVERS)) {
+ scanServers();
+ } else if (event.getPath().contains(Constants.ZTSERVERS)) {
+ int pos = event.getPath().lastIndexOf('/');
+
+ // do only if ZTSERVER is parent
+ if (pos >= 0 && event.getPath().substring(0, pos).endsWith(Constants.ZTSERVERS)) {
+
+ String server = event.getPath().substring(pos + 1);
+
+ final Set<TServerInstance> updates = new HashSet<TServerInstance>();
+ final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
+
+ final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+
+ try {
+ checkServer(updates, doomed, path, server);
+ if (!doomed.isEmpty() || !updates.isEmpty())
+ this.cback.update(this, doomed, updates);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }
+ }
+
+ public synchronized TServerConnection getConnection(TServerInstance server) {
+ if (server == null)
+ return null;
+ TServerInfo tServerInfo = currentInstances.get(server);
+ if (tServerInfo == null)
+ return null;
+ return tServerInfo.connection;
+ }
+
+ public synchronized Set<TServerInstance> getCurrentServers() {
+ return new HashSet<TServerInstance>(currentInstances.keySet());
+ }
+
+ public synchronized int size() {
+ return current.size();
+ }
+
+ public synchronized TServerInstance find(String tabletServer) {
+ HostAndPort addr = AddressUtil.parseAddress(tabletServer, false);
+ for (Entry<String,TServerInfo> entry : current.entrySet()) {
+ if (entry.getValue().instance.getLocation().equals(addr))
+ return entry.getValue().instance;
+ }
+ return null;
+ }
+
+ public synchronized void remove(TServerInstance server) {
+ String zPath = null;
+ for (Entry<String,TServerInfo> entry : current.entrySet()) {
+ if (entry.getValue().instance.equals(server)) {
+ zPath = entry.getKey();
+ break;
+ }
+ }
+ if (zPath == null)
+ return;
+ current.remove(zPath);
+ currentInstances.remove(server);
+
+ log.info("Removing zookeeper lock for " + server);
+ String fullpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + zPath;
+ try {
+ ZooReaderWriter.getInstance().recursiveDelete(fullpath, SKIP);
+ } catch (Exception e) {
+ String msg = "error removing tablet server lock";
+ log.fatal(msg, e);
+ Halt.halt(msg, -1);
+ }
+ getZooCache().clear(fullpath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
index 2f657c4,0000000..568e6f6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
@@@ -1,90 -1,0 +1,91 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.DeadServer;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+public class DeadServerList {
+ private static final Logger log = Logger.getLogger(DeadServerList.class);
+ private final String path;
+
+ public DeadServerList(String path) {
+ this.path = path;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.mkdirs(path);
+ } catch (Exception ex) {
+ log.error("Unable to make parent directories of " + path, ex);
+ }
+ }
+
+ public List<DeadServer> getList() {
+ List<DeadServer> result = new ArrayList<DeadServer>();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ List<String> children = zoo.getChildren(path);
+ if (children != null) {
+ for (String child : children) {
+ Stat stat = new Stat();
+ byte[] data;
+ try {
+ data = zoo.getData(path + "/" + child, stat);
+ } catch (NoNodeException nne) {
+ // Another thread or process can delete child while this loop is running.
+ // We ignore this error since it's harmless if we miss the deleted server
+ // in the dead server list.
+ continue;
+ }
- DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, Constants.UTF8));
++ DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, UTF_8));
+ result.add(server);
+ }
+ }
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ return result;
+ }
+
+ public void delete(String server) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.recursiveDelete(path + "/" + server, NodeMissingPolicy.SKIP);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ public void post(String server, String cause) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
- zoo.putPersistentData(path + "/" + server, cause.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++ zoo.putPersistentData(path + "/" + server, cause.getBytes(UTF_8), NodeExistsPolicy.SKIP);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
index 400d0ac,0000000..b4ef519
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
@@@ -1,143 -1,0 +1,144 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * A tablet is assigned to a tablet server at the given address as long as it is alive and well. When the tablet server is restarted, the instance information
+ * it advertises will change. Therefore tablet assignments can be considered out-of-date if the tablet server instance information has been changed.
+ *
+ */
+public class TServerInstance implements Comparable<TServerInstance>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // HostAndPort is not Serializable
+ private transient HostAndPort location;
+ private String session;
+ private String cachedStringRepresentation;
+
+ public TServerInstance(HostAndPort address, String session) {
+ this.location = address;
+ this.session = session;
+ this.cachedStringRepresentation = hostPort() + "[" + session + "]";
+ }
+
+ public TServerInstance(HostAndPort address, long session) {
+ this(address, Long.toHexString(session));
+ }
+
+ public TServerInstance(String address, long session) {
+ this(AddressUtil.parseAddress(address, false), Long.toHexString(session));
+ }
+
+ public TServerInstance(Value address, Text session) {
- this(AddressUtil.parseAddress(new String(address.get(), Constants.UTF8), false), session.toString());
++ this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString());
+ }
+
+ public void putLocation(Mutation m) {
+ m.put(TabletsSection.CurrentLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
+ }
+
+ public void putFutureLocation(Mutation m) {
+ m.put(TabletsSection.FutureLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
+ }
+
+ public void putLastLocation(Mutation m) {
+ m.put(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
+ }
+
+ public void clearLastLocation(Mutation m) {
+ m.putDelete(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier());
+ }
+
+ @Override
+ public int compareTo(TServerInstance other) {
+ if (this == other)
+ return 0;
+ return this.toString().compareTo(other.toString());
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof TServerInstance) {
+ return compareTo((TServerInstance) obj) == 0;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return cachedStringRepresentation;
+ }
+
+ public int port() {
+ return getLocation().getPort();
+ }
+
+ public String host() {
+ return getLocation().getHostText();
+ }
+
+ public String hostPort() {
+ return getLocation().toString();
+ }
+
+ public Text asColumnQualifier() {
+ return new Text(this.getSession());
+ }
+
+ public Value asMutationValue() {
- return new Value(getLocation().toString().getBytes(Constants.UTF8));
++ return new Value(getLocation().toString().getBytes(UTF_8));
+ }
+
+ public HostAndPort getLocation() {
+ return location;
+ }
+
+ public String getSession() {
+ return session;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.defaultWriteObject();
+ out.writeObject(location.toString());
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ location = HostAndPort.fromString(in.readObject().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
index f4d5591,0000000..91be9c3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
@@@ -1,189 -1,0 +1,190 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SkippingIterator;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+
+public class TabletStateChangeIterator extends SkippingIterator {
+
+ private static final String SERVERS_OPTION = "servers";
+ private static final String TABLES_OPTION = "tables";
+ private static final String MERGES_OPTION = "merges";
+ // private static final Logger log = Logger.getLogger(TabletStateChangeIterator.class);
+
+ Set<TServerInstance> current;
+ Set<String> onlineTables;
+ Map<Text,MergeInfo> merges;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ current = parseServers(options.get(SERVERS_OPTION));
+ onlineTables = parseTables(options.get(TABLES_OPTION));
+ merges = parseMerges(options.get(MERGES_OPTION));
+ }
+
+ private Set<String> parseTables(String tables) {
+ if (tables == null)
+ return null;
+ Set<String> result = new HashSet<String>();
+ for (String table : tables.split(","))
+ result.add(table);
+ return result;
+ }
+
+ private Set<TServerInstance> parseServers(String servers) {
+ if (servers == null)
+ return null;
+ // parse "host:port[INSTANCE]"
+ Set<TServerInstance> result = new HashSet<TServerInstance>();
+ if (servers.length() > 0) {
+ for (String part : servers.split(",")) {
+ String parts[] = part.split("\\[", 2);
+ String hostport = parts[0];
+ String instance = parts[1];
+ if (instance != null && instance.endsWith("]"))
+ instance = instance.substring(0, instance.length() - 1);
+ result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance));
+ }
+ }
+ return result;
+ }
+
+ private Map<Text,MergeInfo> parseMerges(String merges) {
+ if (merges == null)
+ return null;
+ try {
+ Map<Text,MergeInfo> result = new HashMap<Text,MergeInfo>();
+ DataInputBuffer buffer = new DataInputBuffer();
- byte[] data = Base64.decodeBase64(merges.getBytes(Constants.UTF8));
++ byte[] data = Base64.decodeBase64(merges.getBytes(UTF_8));
+ buffer.reset(data, data.length);
+ while (buffer.available() > 0) {
+ MergeInfo mergeInfo = new MergeInfo();
+ mergeInfo.readFields(buffer);
+ result.put(mergeInfo.extent.getTableId(), mergeInfo);
+ }
+ return result;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ protected void consume() throws IOException {
+ while (getSource().hasTop()) {
+ Key k = getSource().getTopKey();
+ Value v = getSource().getTopValue();
+
+ if (onlineTables == null || current == null)
+ return;
+
+ TabletLocationState tls;
+ try {
+ tls = MetaDataTableScanner.createTabletLocationState(k, v);
+ if (tls == null)
+ return;
+ } catch (BadLocationStateException e) {
+ // maybe the master can do something with a tablet with bad/inconsistent state
+ return;
+ }
+ // we always want data about merges
+ MergeInfo merge = merges.get(tls.extent.getTableId());
+ if (merge != null && merge.getExtent() != null && merge.getExtent().overlaps(tls.extent)) {
+ return;
+ }
+ // is the table supposed to be online or offline?
+ boolean shouldBeOnline = onlineTables.contains(tls.extent.getTableId().toString());
+
+ switch (tls.getState(current)) {
+ case ASSIGNED:
+ // we always want data about assigned tablets
+ return;
+ case HOSTED:
+ if (!shouldBeOnline)
+ return;
+ case ASSIGNED_TO_DEAD_SERVER:
+ return;
+ case UNASSIGNED:
+ if (shouldBeOnline)
+ return;
+ }
+ // table is in the expected state so don't bother returning any information about it
+ getSource().next();
+ }
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ public static void setCurrentServers(IteratorSetting cfg, Set<TServerInstance> goodServers) {
+ if (goodServers != null) {
+ List<String> servers = new ArrayList<String>();
+ for (TServerInstance server : goodServers)
+ servers.add(server.toString());
+ cfg.addOption(SERVERS_OPTION, StringUtil.join(servers, ","));
+ }
+ }
+
+ public static void setOnlineTables(IteratorSetting cfg, Set<String> onlineTables) {
+ if (onlineTables != null)
+ cfg.addOption(TABLES_OPTION, StringUtil.join(onlineTables, ","));
+ }
+
+ public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) {
+ DataOutputBuffer buffer = new DataOutputBuffer();
+ try {
+ for (MergeInfo info : merges) {
+ KeyExtent extent = info.getExtent();
+ if (extent != null && !info.getState().equals(MergeState.NONE)) {
+ info.write(buffer);
+ }
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ String encoded = Base64.encodeBase64String(Arrays.copyOf(buffer.getData(), buffer.getLength()));
+ cfg.addOption(MERGES_OPTION, encoded);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
index b0ed03f,0000000..8f0c9c7
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
@@@ -1,97 -1,0 +1,98 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+
+public class ZooStore implements DistributedStore {
+
+ private static final Logger log = Logger.getLogger(ZooStore.class);
+
+ String basePath;
+
+ ZooCache cache = new ZooCache();
+
+ public ZooStore(String basePath) throws IOException {
+ if (basePath.endsWith("/"))
+ basePath = basePath.substring(0, basePath.length() - 1);
+ this.basePath = basePath;
+ }
+
+ public ZooStore() throws IOException {
+ this(ZooUtil.getRoot(HdfsZooInstance.getInstance().getInstanceID()));
+ }
+
+ @Override
+ public byte[] get(String path) throws DistributedStoreException {
+ try {
+ return cache.get(relative(path));
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ private String relative(String path) {
+ return basePath + path;
+ }
+
+ @Override
+ public List<String> getChildren(String path) throws DistributedStoreException {
+ try {
+ return cache.getChildren(relative(path));
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public void put(String path, byte[] bs) throws DistributedStoreException {
+ try {
+ path = relative(path);
+ ZooReaderWriter.getInstance().putPersistentData(path, bs, NodeExistsPolicy.OVERWRITE);
+ cache.clear();
- log.debug("Wrote " + new String(bs, Constants.UTF8) + " to " + path);
++ log.debug("Wrote " + new String(bs, UTF_8) + " to " + path);
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public void remove(String path) throws DistributedStoreException {
+ try {
+ log.debug("Removing " + path);
+ path = relative(path);
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.exists(path))
+ zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ cache.clear();
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
index 0a0e1ee,0000000..15ade65
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
@@@ -1,179 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.log4j.Logger;
+
+import com.google.common.net.HostAndPort;
+
+public class ZooTabletStateStore extends TabletStateStore {
+
+ private static final Logger log = Logger.getLogger(ZooTabletStateStore.class);
+ final private DistributedStore store;
+
+ public ZooTabletStateStore(DistributedStore store) {
+ this.store = store;
+ }
+
+ public ZooTabletStateStore() throws DistributedStoreException {
+ try {
+ store = new ZooStore();
+ } catch (IOException ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public ClosableIterator<TabletLocationState> iterator() {
+ return new ClosableIterator<TabletLocationState>() {
+ boolean finished = false;
+
+ @Override
+ public boolean hasNext() {
+ return !finished;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ finished = true;
+ try {
+ byte[] future = store.get(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
+ byte[] current = store.get(RootTable.ZROOT_TABLET_LOCATION);
+ byte[] last = store.get(RootTable.ZROOT_TABLET_LAST_LOCATION);
+
+ TServerInstance currentSession = null;
+ TServerInstance futureSession = null;
+ TServerInstance lastSession = null;
+
+ if (future != null)
+ futureSession = parse(future);
+
+ if (last != null)
+ lastSession = parse(last);
+
+ if (current != null) {
+ currentSession = parse(current);
+ futureSession = null;
+ }
+ List<Collection<String>> logs = new ArrayList<Collection<String>>();
+ for (String entry : store.getChildren(RootTable.ZROOT_TABLET_WALOGS)) {
+ byte[] logInfo = store.get(RootTable.ZROOT_TABLET_WALOGS + "/" + entry);
+ if (logInfo != null) {
+ LogEntry logEntry = new LogEntry();
+ logEntry.fromBytes(logInfo);
+ logs.add(logEntry.logSet);
+ log.debug("root tablet logSet " + logEntry.logSet);
+ }
+ }
+ TabletLocationState result = new TabletLocationState(RootTable.EXTENT, futureSession, currentSession, lastSession, logs, false);
+ log.debug("Returning root tablet state: " + result);
+ return result;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void close() {
+ }
+ };
+ }
+
+ protected TServerInstance parse(byte[] current) {
- String str = new String(current, Constants.UTF8);
++ String str = new String(current, UTF_8);
+ String[] parts = str.split("[|]", 2);
+ HostAndPort address = HostAndPort.fromString(parts[0]);
+ if (parts.length > 1 && parts[1] != null && parts[1].length() > 0) {
+ return new TServerInstance(address, parts[1]);
+ } else {
+ // a 1.2 location specification: DO NOT WANT
+ return null;
+ }
+ }
+
+ @Override
+ public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException {
+ if (assignments.size() != 1)
+ throw new IllegalArgumentException("There is only one root tablet");
+ Assignment assignment = assignments.iterator().next();
+ if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
+ throw new IllegalArgumentException("You can only store the root tablet location");
+ String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
+ Iterator<TabletLocationState> currentIter = iterator();
+ TabletLocationState current = currentIter.next();
+ if (current.current != null) {
+ throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
+ }
- store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(Constants.UTF8));
++ store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(UTF_8));
+ }
+
+ @Override
+ public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException {
+ if (assignments.size() != 1)
+ throw new IllegalArgumentException("There is only one root tablet");
+ Assignment assignment = assignments.iterator().next();
+ if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
+ throw new IllegalArgumentException("You can only store the root tablet location");
+ String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
+ Iterator<TabletLocationState> currentIter = iterator();
+ TabletLocationState current = currentIter.next();
+ if (current.current != null) {
+ throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
+ }
+ if (!current.future.equals(assignment.server)) {
+ throw new DistributedStoreException("Root tablet is already assigned to " + current.future);
+ }
- store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(Constants.UTF8));
- store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(Constants.UTF8));
++ store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(UTF_8));
++ store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(UTF_8));
+ // Make the following unnecessary by making the entire update atomic
+ store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
+ log.debug("Put down root tablet location");
+ }
+
+ @Override
+ public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException {
+ if (tablets.size() != 1)
+ throw new IllegalArgumentException("There is only one root tablet");
+ TabletLocationState tls = tablets.iterator().next();
+ if (tls.extent.compareTo(RootTable.EXTENT) != 0)
+ throw new IllegalArgumentException("You can only store the root tablet location");
+ store.remove(RootTable.ZROOT_TABLET_LOCATION);
+ store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
+ log.debug("unassign root tablet location");
+ }
+
+ @Override
+ public String name() {
+ return "Root Table";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 16b4bb8,0000000..409fa62
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,273 -1,0 +1,274 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
- logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
++ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), UTF_8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
index 0a5341a,0000000..4e909c0
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
@@@ -1,187 -1,0 +1,189 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.monitor;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.net.SocketNode;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Hijack log4j and capture log events for display.
+ *
+ */
+public class LogService extends org.apache.log4j.AppenderSkeleton {
+
+ private static final Logger log = Logger.getLogger(LogService.class);
+
+ /**
+ * Read logging events forward to us over the net.
+ *
+ */
+ static class SocketServer implements Runnable {
+ private ServerSocket server = null;
+
+ public SocketServer(int port) {
+ try {
+ server = new ServerSocket(port);
+ } catch (IOException io) {
+ throw new RuntimeException(io);
+ }
+ }
+
+ public int getLocalPort() {
+ return server.getLocalPort();
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ log.debug("Waiting for log message senders");
+ Socket socket = server.accept();
+ log.debug("Got a new connection");
+ Thread t = new Daemon(new SocketNode(socket, LogManager.getLoggerRepository()));
+ t.start();
+ }
+ } catch (IOException io) {
+ log.error(io, io);
+ }
+ }
+ }
+
+ /**
+ * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
+ *
+ * @param conf
+ * configuration for the instance
+ * @param instanceId
+ * instanceId for the instance
+ * @param hostAddress
+ * Address that monitor process is bound to
+ */
+ public static void startLogListener(AccumuloConfiguration conf, String instanceId, String hostAddress) {
+ try {
+ SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));
+
+ // getLocalPort will return the actual ephemeral port used when '0' was provided.
+ String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
+
+ log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);
+
+ final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
+ final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ // Delete before we try to re-create in case the previous session hasn't yet expired
+ try {
+ zoo.delete(path, -1);
+ } catch (KeeperException e) {
+ // We don't care if the node is already gone
+ if (!KeeperException.Code.NONODE.equals(e.code())) {
+ throw e;
+ }
+ }
+
- zoo.putEphemeralData(path, logForwardingAddr.getBytes(Constants.UTF8));
++ zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
+
+ new Daemon(server).start();
+ } catch (Throwable t) {
+ log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
+ }
+ }
+
+ static private LogService instance = null;
+
+ synchronized public static LogService getInstance() {
+ if (instance == null)
+ return new LogService();
+ return instance;
+ }
+
+ private static final int MAX_LOGS = 50;
+
+ private LinkedHashMap<String,DedupedLogEvent> events = new LinkedHashMap<String,DedupedLogEvent>(MAX_LOGS + 1, (float) .75, true) {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ protected boolean removeEldestEntry(Map.Entry eldest) {
+ return size() > MAX_LOGS;
+ }
+ };
+
+ public LogService() {
+ synchronized (LogService.class) {
+ instance = this;
+ }
+ }
+
+ @Override
+ synchronized protected void append(LoggingEvent ev) {
+ Object application = ev.getMDC("application");
+ if (application == null || application.toString().isEmpty())
+ return;
+
+ DedupedLogEvent dev = new DedupedLogEvent(ev);
+
+ // if event is present, increase the count
+ if (events.containsKey(dev.toString())) {
+ DedupedLogEvent oldDev = events.remove(dev.toString());
+ dev.setCount(oldDev.getCount() + 1);
+ }
+ events.put(dev.toString(), dev);
+ }
+
+ @Override
+ public void close() {
+ events = null;
+ }
+
+ @Override
+ public synchronized void doAppend(LoggingEvent event) {
+ super.doAppend(event);
+ }
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ synchronized public List<DedupedLogEvent> getEvents() {
+ return new ArrayList<DedupedLogEvent>(events.values());
+ }
+
+ synchronized public void clear() {
+ events.clear();
+ }
+}
[26/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 03fe069,0000000..94be0bb
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -1,3928 -1,0 +1,3929 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.CompressedIterators;
+import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
+import org.apache.accumulo.core.client.impl.Translator.TKeyExtentTranslator;
+import org.apache.accumulo.core.client.impl.Translator.TRangeTranslator;
+import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TCMResult;
+import org.apache.accumulo.core.data.thrift.TCMStatus;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TCondition;
+import org.apache.accumulo.core.data.thrift.TConditionalMutation;
+import org.apache.accumulo.core.data.thrift.TConditionalSession;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.AuthorizationContainer;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.log.SortedLogState;
+import org.apache.accumulo.server.master.recovery.RecoveryPath;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.RpcWrapper;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.tserver.Compactor.CompactionInfo;
+import org.apache.accumulo.tserver.RowLocks.RowLock;
+import org.apache.accumulo.tserver.Tablet.CommitSession;
+import org.apache.accumulo.tserver.Tablet.KVEntry;
+import org.apache.accumulo.tserver.Tablet.LookupResult;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.Tablet.ScanBatch;
+import org.apache.accumulo.tserver.Tablet.Scanner;
+import org.apache.accumulo.tserver.Tablet.SplitInfo;
+import org.apache.accumulo.tserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.tserver.Tablet.TabletClosedException;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.data.ServerConditionalMutation;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.LogSorter;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.log.TabletServerLogger;
+import org.apache.accumulo.tserver.mastermessage.MasterMessage;
+import org.apache.accumulo.tserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMBean;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.net.HostAndPort;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.tserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long lastMemoryCheckTime = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+ private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000;
+ private static final long TIME_BETWEEN_GC_CHECKS = 5000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, VolumeManager fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = fs;
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, TIME_BETWEEN_GC_CHECKS, TIME_BETWEEN_GC_CHECKS);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ long now = System.currentTimeMillis();
+
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) {
+ long diff = now - lastMemoryCheckTime;
+ if (diff > keepAliveTimeout) {
+ log.warn(String.format("GC pause checker not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check",
+ TIME_BETWEEN_GC_CHECKS / 1000., diff / 1000.));
+ }
+ lastMemoryCheckTime = now;
+ return;
+ }
+
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ lastMemoryCheckTime = now;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+ long maxIdle;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ long getMaxIdleTime() {
+ return maxIdle;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
+ */
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized Session reserveSession(long sessionId, boolean wait) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ while (wait && session.reserved) {
+ try {
+ wait(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ notifyAll();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ return removeSession(sessionId, false);
+ }
+
+ Session removeSession(long sessionId, boolean unreserve) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ if (unreserve && session != null)
+ unreserveSession(session);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ @Deprecated
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ @Override
+ public AuthorizationContainer getAuthorizationsContainer() {
+ return new AuthorizationContainer() {
+ @Override
+ public boolean contains(ByteSequence auth) {
+ try {
+ return security.userHasAuthorizations(credentials,
+ Collections.<ByteBuffer> singletonList(ByteBuffer.wrap(auth.getBackingArray(), auth.offset(), auth.length())));
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class ConditionalSession extends Session {
+ public TCredentials credentials;
+ public Authorizations auths;
+ public String tableId;
+ public AtomicBoolean interruptFlag;
+
+ @Override
+ public void cleanup() {
+ interruptFlag.set(true);
+ }
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+ public long readaheadThreshold = Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
+ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ private RowLocks rowLocks = new RowLocks();
+
+ ThriftClientHandler() {
+ super(instance, watcher, fs);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+ Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
+ for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
+ Path path = new Path(mapping.getKey());
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ path = ns.makeQualified(path);
+ fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
+ }
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileRefMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
+ long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
- String tableId = new String(textent.getTable(), Constants.UTF8);
++ String tableId = new String(textent.getTable(), UTF_8);
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // metadata
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+ scanSession.readaheadThreshold = readaheadThreshold;
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
- tables.add(new String(keyExtent.getTable(), Constants.UTF8));
++ tables.add(new String(keyExtent.getTable(), UTF_8));
+ }
+
+ if (tables.size() != 1)
+ throw new IllegalArgumentException("Cannot batch scan over multiple tables");
+
+ // check if user has permission to the tables
+ for (String tableId : tables)
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), tbatch, tcolumns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ try {
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+ } catch (ThriftSecurityException tse) {
+ log.error(tse, tse);
+ throw tse;
+ }
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, new TKeyExtentTranslator(), new Translator.ListTranslator<TRange,Range>(
+ new TRangeTranslator()));
+
+ // This is used to determine which thread pool to use
+ KeyExtent threadPoolExtent = batch.keySet().iterator().next();
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ String tableId = keyExtent.getTableId().toString();
+ if (sameTable || security.canWrite(us.credentials, tableId, Tables.getNamespaceId(instance, tableId))) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ try {
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ } finally {
+ prep.stop();
+ }
+
+ long pt2 = System.currentTimeMillis();
+ us.prepareTimes.addStat(pt2 - pt1);
+ updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ updateWalogWriteTime((t2 - t1));
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ wal.stop();
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commit.stop();
+ }
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ private void updateWalogWriteTime(long time) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, time);
+ }
+
+ private void updateAvgCommitTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, (long) ((time) / (double) size));
+ }
+
+ private void updateAvgPrepTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (long) ((time) / (double) size));
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
- String tableId = new String(tkeyExtent.getTable(), Constants.UTF8);
++ String tableId = new String(tkeyExtent.getTable(), UTF_8);
+ if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs;
+ try {
+ cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ } finally {
+ prep.stop();
+ }
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ } finally {
+ wal.stop();
+ }
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ cs.commit(mutations);
+ } finally {
+ commit.stop();
+ }
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ private void checkConditions(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs,
+ List<String> symbols) throws IOException {
+ Iterator<Entry<KeyExtent,List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
+
+ CompressedIterators compressedIters = new CompressedIterators(symbols);
+
+ while (iter.hasNext()) {
+ Entry<KeyExtent,List<ServerConditionalMutation>> entry = iter.next();
+ Tablet tablet = onlineTablets.get(entry.getKey());
+
+ if (tablet == null || tablet.isClosed()) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ iter.remove();
+ } else {
+ List<ServerConditionalMutation> okMutations = new ArrayList<ServerConditionalMutation>(entry.getValue().size());
+
+ for (ServerConditionalMutation scm : entry.getValue()) {
+ if (checkCondition(results, cs, compressedIters, tablet, scm))
+ okMutations.add(scm);
+ }
+
+ entry.setValue(okMutations);
+ }
+
+ }
+ }
+
+ boolean checkCondition(ArrayList<TCMResult> results, ConditionalSession cs, CompressedIterators compressedIters, Tablet tablet,
+ ServerConditionalMutation scm) throws IOException {
+ boolean add = true;
+
+ Set<Column> emptyCols = Collections.emptySet();
+
+ for (TCondition tc : scm.getConditions()) {
+
+ Range range;
+ if (tc.hasTimestamp)
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()), tc.getTs());
+ else
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()));
+
+ IterConfig ic = compressedIters.decompress(tc.iterators);
+
+ Scanner scanner = tablet.createScanner(range, 1, emptyCols, cs.auths, ic.ssiList, ic.ssio, false, cs.interruptFlag);
+
+ try {
+ ScanBatch batch = scanner.read();
+
+ Value val = null;
+
+ for (KVEntry entry2 : batch.results) {
+ val = entry2.getValue();
+ break;
+ }
+
+ if ((val == null ^ tc.getVal() == null) || (val != null && !Arrays.equals(tc.getVal(), val.get()))) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.REJECTED));
+ add = false;
+ break;
+ }
+
+ } catch (TabletClosedException e) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (IterationInterruptedException iie) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (TooManyFilesException tmfe) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ }
+ }
+ return add;
+ }
+
+ private void writeConditionalMutations(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
+ Set<Entry<KeyExtent,List<ServerConditionalMutation>>> es = updates.entrySet();
+
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+
+ boolean sessionCanceled = sess.interruptFlag.get();
+
+ Span prepSpan = Trace.start("prep");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<KeyExtent,List<ServerConditionalMutation>> entry : es) {
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null || tablet.isClosed() || sessionCanceled) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ try {
+
+ @SuppressWarnings("unchecked")
+ List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
+ if (mutations.size() > 0) {
+
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, sess.credentials), mutations);
+
+ if (cs == null) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.ACCEPTED));
+ sendables.put(cs, mutations);
+ }
+ }
+ } catch (TConstraintViolationException e) {
+ if (e.getNonViolators().size() > 0) {
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ for (Mutation m : e.getNonViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.ACCEPTED));
+ }
+
+ for (Mutation m : e.getViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.VIOLATED));
+ }
+ }
+ }
+
+ long t2 = System.currentTimeMillis();
+ updateAvgPrepTime(t2 - t1, es.size());
+ } finally {
+ prepSpan.stop();
+ }
+
+ Span walSpan = Trace.start("wal");
+ try {
+ while (true && sendables.size() > 0) {
+ try {
+ long t1 = System.currentTimeMillis();
+ logger.logManyTablets(sendables);
+ long t2 = System.currentTimeMillis();
+ updateWalogWriteTime(t2 - t1);
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ walSpan.stop();
+ }
+
+ Span commitSpan = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+ }
+ long t2 = System.currentTimeMillis();
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commitSpan.stop();
+ }
+
+ }
+
+ private Map<KeyExtent,List<ServerConditionalMutation>> conditionalUpdate(ConditionalSession cs, Map<KeyExtent,List<ServerConditionalMutation>> updates,
+ ArrayList<TCMResult> results, List<String> symbols) throws IOException {
+ // sort each list of mutations, this is done to avoid deadlock and doing seeks in order is more efficient and detect duplicate rows.
+ ConditionalMutationSet.sortConditionalMutations(updates);
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = new HashMap<KeyExtent,List<ServerConditionalMutation>>();
+
+ // can not process two mutations for the same row, because one will not see what the other writes
+ ConditionalMutationSet.deferDuplicatesRows(updates, deferred);
+
+ // get as many locks as possible w/o blocking... defer any rows that are locked
+ List<RowLock> locks = rowLocks.acquireRowlocks(updates, deferred);
+ try {
+ Span checkSpan = Trace.start("Check conditions");
+ try {
+ checkConditions(updates, results, cs, symbols);
+ } finally {
+ checkSpan.stop();
+ }
+
+ Span updateSpan = Trace.start("apply conditional mutations");
+ try {
+ writeConditionalMutations(updates, results, cs);
+ } finally {
+ updateSpan.stop();
+ }
+ } finally {
+ rowLocks.releaseRowLocks(locks);
+ }
+ return deferred;
+ }
+
+ @Override
+ public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials credentials, List<ByteBuffer> authorizations, String tableId)
+ throws ThriftSecurityException, TException {
+
+ Authorizations userauths = null;
+ if (!security.canConditionallyUpdate(credentials, tableId, Tables.getNamespaceId(instance, tableId), authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ ConditionalSession cs = new ConditionalSession();
+ cs.auths = new Authorizations(authorizations);
+ cs.credentials = credentials;
+ cs.tableId = tableId;
+ cs.interruptFlag = new AtomicBoolean();
+
+ long sid = sessionManager.createSession(cs, false);
+ return new TConditionalSession(sid, lockID, sessionManager.getMaxIdleTime());
+ }
+
+ @Override
+ public List<TCMResult> conditionalUpdate(TInfo tinfo, long sessID, Map<TKeyExtent,List<TConditionalMutation>> mutations, List<String> symbols)
+ throws NoSuchScanIDException, TException {
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.reserveSession(sessID);
+
+ if (cs == null || cs.interruptFlag.get())
+ throw new NoSuchScanIDException();
+
+ if (!cs.tableId.equals(MetadataTable.ID) && !cs.tableId.equals(RootTable.ID))
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Text tid = new Text(cs.tableId);
+ long opid = writeTracker.startWrite(TabletType.type(new KeyExtent(tid, null, null)));
+
+ try {
+ Map<KeyExtent,List<ServerConditionalMutation>> updates = Translator.translate(mutations, Translators.TKET,
+ new Translator.ListTranslator<TConditionalMutation,ServerConditionalMutation>(ServerConditionalMutation.TCMT));
+
+ for (KeyExtent ke : updates.keySet())
+ if (!ke.getTableId().equals(tid))
+ throw new IllegalArgumentException("Unexpected table id " + tid + " != " + ke.getTableId());
+
+ ArrayList<TCMResult> results = new ArrayList<TCMResult>();
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = conditionalUpdate(cs, updates, results, symbols);
+
+ while (deferred.size() > 0) {
+ deferred = conditionalUpdate(cs, deferred, results, symbols);
+ }
+
+ return results;
+ } catch (IOException ioe) {
+ throw new TException(ioe);
+ } finally {
+ writeTracker.finishWrite(opid);
+ sessionManager.unreserveSession(sessID);
+ }
+ }
+
+ @Override
+ public void invalidateConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ // this method should wait for any running conditional update to complete
+ // after this method returns a conditional update should not be able to start
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.getSession(sessID);
+ if (cs != null)
+ cs.interruptFlag.set(true);
+
+ cs = (ConditionalSession) sessionManager.reserveSession(sessID, true);
+ if (cs != null)
+ sessionManager.removeSession(sessID, true);
+ }
+
+ @Override
+ public void closeConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ sessionManager.removeSession(sessID, false);
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
+ ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table));
+ String namespaceId;
+ try {
+ namespaceId = Tables.getNamespaceId(instance, tableId);
+ } catch (IllegalArgumentException ex) {
+ // table does not exist, try to educate the client
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!security.canSplitTablet(credentials, tableId, namespaceId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, final String request) throws ThriftSecurityException {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName())) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configu
<TRUNCATED>
[06/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index 0778f5b,0000000..3459069
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@@ -1,3857 -1,0 +1,3859 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSource;
+import org.apache.accumulo.core.iterators.system.StatsIterator;
+import org.apache.accumulo.core.iterators.system.VisibilityFilter;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.tableOps.CompactionIterators;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.FileUtil;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.TabletOperations;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.tserver.Compactor.CompactionCanceledException;
+import org.apache.accumulo.tserver.Compactor.CompactionEnv;
+import org.apache.accumulo.tserver.FileManager.ScanFileManager;
+import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
+import org.apache.accumulo.tserver.TabletServer.TservConstraintEnv;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.apache.accumulo.tserver.compaction.WriteParameters;
+import org.apache.accumulo.tserver.constraints.ConstraintChecker;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/*
+ * We need to be able to have the master tell a tabletServer to
+ * close this file, and the tablet server to handle all pending client reads
+ * before closing
+ *
+ */
+
+/**
+ *
+ * this class just provides an interface to read from a MapFile mostly takes care of reporting start and end keys
+ *
+ * need this because a single row extent can have multiple columns this manages all the columns (each handled by a store) for a single row-extent
+ *
+ *
+ */
+
+public class Tablet {
+
+ enum MinorCompactionReason {
+ USER, SYSTEM, CLOSE, RECOVERY
+ }
+
+ public class CommitSession {
+
+ private int seq;
+ private InMemoryMap memTable;
+ private int commitsInProgress;
+ private long maxCommittedTime = Long.MIN_VALUE;
+
+ private CommitSession(int seq, InMemoryMap imm) {
+ this.seq = seq;
+ this.memTable = imm;
+ commitsInProgress = 0;
+ }
+
+ public int getWALogSeq() {
+ return seq;
+ }
+
+ private void decrementCommitsInProgress() {
+ if (commitsInProgress < 1)
+ throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
+
+ commitsInProgress--;
+ if (commitsInProgress == 0)
+ Tablet.this.notifyAll();
+ }
+
+ private void incrementCommitsInProgress() {
+ if (commitsInProgress < 0)
+ throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
+
+ commitsInProgress++;
+ }
+
+ private void waitForCommitsToFinish() {
+ while (commitsInProgress > 0) {
+ try {
+ Tablet.this.wait(50);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ public void abortCommit(List<Mutation> value) {
+ Tablet.this.abortCommit(this, value);
+ }
+
+ public void commit(List<Mutation> mutations) {
+ Tablet.this.commit(this, mutations);
+ }
+
+ public Tablet getTablet() {
+ return Tablet.this;
+ }
+
+ public boolean beginUpdatingLogsUsed(ArrayList<DfsLogger> copy, boolean mincFinish) {
+ return Tablet.this.beginUpdatingLogsUsed(memTable, copy, mincFinish);
+ }
+
+ public void finishUpdatingLogsUsed() {
+ Tablet.this.finishUpdatingLogsUsed();
+ }
+
+ public int getLogId() {
+ return logId;
+ }
+
+ public KeyExtent getExtent() {
+ return extent;
+ }
+
+ private void updateMaxCommittedTime(long time) {
+ maxCommittedTime = Math.max(time, maxCommittedTime);
+ }
+
+ private long getMaxCommittedTime() {
+ if (maxCommittedTime == Long.MIN_VALUE)
+ throw new IllegalStateException("Tried to read max committed time when it was never set");
+ return maxCommittedTime;
+ }
+
+ }
+
+ private class TabletMemory {
+ private InMemoryMap memTable;
+ private InMemoryMap otherMemTable;
+ private InMemoryMap deletingMemTable;
+ private int nextSeq = 1;
+ private CommitSession commitSession;
+
+ TabletMemory() {
+ try {
+ memTable = new InMemoryMap(acuTableConf);
+ } catch (LocalityGroupConfigurationError e) {
+ throw new RuntimeException(e);
+ }
+ commitSession = new CommitSession(nextSeq, memTable);
+ nextSeq += 2;
+ }
+
+ InMemoryMap getMemTable() {
+ return memTable;
+ }
+
+ InMemoryMap getMinCMemTable() {
+ return otherMemTable;
+ }
+
+ CommitSession prepareForMinC() {
+ if (otherMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ if (deletingMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ otherMemTable = memTable;
+ try {
+ memTable = new InMemoryMap(acuTableConf);
+ } catch (LocalityGroupConfigurationError e) {
+ throw new RuntimeException(e);
+ }
+
+ CommitSession oldCommitSession = commitSession;
+ commitSession = new CommitSession(nextSeq, memTable);
+ nextSeq += 2;
+
+ tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), otherMemTable.estimatedSizeInBytes());
+
+ return oldCommitSession;
+ }
+
+ void finishedMinC() {
+
+ if (otherMemTable == null) {
+ throw new IllegalStateException();
+ }
+
+ if (deletingMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ deletingMemTable = otherMemTable;
+
+ otherMemTable = null;
+ Tablet.this.notifyAll();
+ }
+
+ void finalizeMinC() {
+ try {
+ deletingMemTable.delete(15000);
+ } finally {
+ synchronized (Tablet.this) {
+ if (otherMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ if (deletingMemTable == null) {
+ throw new IllegalStateException();
+ }
+
+ deletingMemTable = null;
+
+ tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), 0);
+ }
+ }
+ }
+
+ boolean memoryReservedForMinC() {
+ return otherMemTable != null || deletingMemTable != null;
+ }
+
+ void waitForMinC() {
+ while (otherMemTable != null || deletingMemTable != null) {
+ try {
+ Tablet.this.wait(50);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ void mutate(CommitSession cm, List<Mutation> mutations) {
+ cm.memTable.mutate(mutations);
+ }
+
+ void updateMemoryUsageStats() {
+ long other = 0;
+ if (otherMemTable != null)
+ other = otherMemTable.estimatedSizeInBytes();
+ else if (deletingMemTable != null)
+ other = deletingMemTable.estimatedSizeInBytes();
+
+ tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), other);
+ }
+
+ List<MemoryIterator> getIterators() {
+ List<MemoryIterator> toReturn = new ArrayList<MemoryIterator>(2);
+ toReturn.add(memTable.skvIterator());
+ if (otherMemTable != null)
+ toReturn.add(otherMemTable.skvIterator());
+ return toReturn;
+ }
+
+ void returnIterators(List<MemoryIterator> iters) {
+ for (MemoryIterator iter : iters) {
+ iter.close();
+ }
+ }
+
+ public long getNumEntries() {
+ if (otherMemTable != null)
+ return memTable.getNumEntries() + otherMemTable.getNumEntries();
+ return memTable.getNumEntries();
+ }
+
+ CommitSession getCommitSession() {
+ return commitSession;
+ }
+ }
+
+ private TabletMemory tabletMemory;
+
+ private final TabletTime tabletTime;
+ private long persistedTime;
+ private final Object timeLock = new Object();
+
+ private final Path location; // absolute path of this tablets dir
+ private TServerInstance lastLocation;
+
+ private Configuration conf;
+ private VolumeManager fs;
+
+ private TableConfiguration acuTableConf;
+
+ private volatile boolean tableDirChecked = false;
+
+ private AtomicLong dataSourceDeletions = new AtomicLong(0);
+ private Set<ScanDataSource> activeScans = new HashSet<ScanDataSource>();
+
+ private volatile boolean closing = false;
+ private boolean closed = false;
+ private boolean closeComplete = false;
+
+ private long lastFlushID = -1;
+ private long lastCompactID = -1;
+
+ private static class CompactionWaitInfo {
+ long flushID = -1;
+ long compactionID = -1;
+ }
+
+ // stores info about user initiated major compaction that is waiting on a minor compaction to finish
+ private CompactionWaitInfo compactionWaitInfo = new CompactionWaitInfo();
+
+ private KeyExtent extent;
+
+ private TabletResourceManager tabletResources;
+ final private DatafileManager datafileManager;
+ private volatile boolean majorCompactionInProgress = false;
+ private volatile boolean majorCompactionWaitingToStart = false;
+ private Set<MajorCompactionReason> majorCompactionQueued = Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
+ private volatile boolean minorCompactionInProgress = false;
+ private volatile boolean minorCompactionWaitingToStart = false;
+
+ private boolean updatingFlushID = false;
+
+ private AtomicReference<ConstraintChecker> constraintChecker = new AtomicReference<ConstraintChecker>();
+
+ private final String tabletDirectory;
+
+ private int writesInProgress = 0;
+
+ private static final Logger log = Logger.getLogger(Tablet.class);
+ public TabletStatsKeeper timer;
+
+ private Rate queryRate = new Rate(0.2);
+ private long queryCount = 0;
+
+ private Rate queryByteRate = new Rate(0.2);
+ private long queryBytes = 0;
+
+ private Rate ingestRate = new Rate(0.2);
+ private long ingestCount = 0;
+
+ private Rate ingestByteRate = new Rate(0.2);
+ private long ingestBytes = 0;
+
+ private byte[] defaultSecurityLabel = new byte[0];
+
+ private long lastMinorCompactionFinishTime;
+ private long lastMapFileImportTime;
+
+ private volatile long numEntries;
+ private volatile long numEntriesInMemory;
+
+ // a count of the amount of data read by the iterators
+ private AtomicLong scannedCount = new AtomicLong(0);
+ private Rate scannedRate = new Rate(0.2);
+
+ private ConfigurationObserver configObserver;
+
+ private TabletServer tabletServer;
+
+ private final int logId;
+ // ensure we only have one reader/writer of our bulk file notes at at time
+ public final Object bulkFileImportLock = new Object();
+
+ public int getLogId() {
+ return logId;
+ }
+
+ public static class TabletClosedException extends RuntimeException {
+ public TabletClosedException(Exception e) {
+ super(e);
+ }
+
+ public TabletClosedException() {
+ super();
+ }
+
+ private static final long serialVersionUID = 1L;
+ }
+
+ FileRef getNextMapFilename(String prefix) throws IOException {
+ String extension = FileOperations.getNewFileExtension(tabletServer.getTableConfiguration(extent));
+ checkTabletDir();
+ return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension);
+ }
+
+ private void checkTabletDir() throws IOException {
+ if (!tableDirChecked) {
+ checkTabletDir(this.location);
+ tableDirChecked = true;
+ }
+ }
+
+ private void checkTabletDir(Path tabletDir) throws IOException {
+
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(tabletDir);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+
+ if (files == null) {
+ if (tabletDir.getName().startsWith(Constants.CLONE_PREFIX))
+ log.debug("Tablet " + extent + " had no dir, creating " + tabletDir); // its a clone dir...
+ else
+ log.warn("Tablet " + extent + " had no dir, creating " + tabletDir);
+
+ fs.mkdirs(tabletDir);
+ }
+ }
+
+ class DatafileManager {
+ // access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
+ final private Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
+
+ DatafileManager(SortedMap<FileRef,DataFileValue> datafileSizes) {
+ for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet())
+ this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
+ }
+
+ FileRef mergingMinorCompactionFile = null;
+ Set<FileRef> filesToDeleteAfterScan = new HashSet<FileRef>();
+ Map<Long,Set<FileRef>> scanFileReservations = new HashMap<Long,Set<FileRef>>();
+ MapCounter<FileRef> fileScanReferenceCounts = new MapCounter<FileRef>();
+ long nextScanReservationId = 0;
+ boolean reservationsBlocked = false;
+
+ Set<FileRef> majorCompactingFiles = new HashSet<FileRef>();
+
+ Pair<Long,Map<FileRef,DataFileValue>> reserveFilesForScan() {
+ synchronized (Tablet.this) {
+
+ while (reservationsBlocked) {
+ try {
+ Tablet.this.wait(50);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+
+ Set<FileRef> absFilePaths = new HashSet<FileRef>(datafileSizes.keySet());
+
+ long rid = nextScanReservationId++;
+
+ scanFileReservations.put(rid, absFilePaths);
+
+ Map<FileRef,DataFileValue> ret = new HashMap<FileRef,DataFileValue>();
+
+ for (FileRef path : absFilePaths) {
+ fileScanReferenceCounts.increment(path, 1);
+ ret.put(path, datafileSizes.get(path));
+ }
+
+ return new Pair<Long,Map<FileRef,DataFileValue>>(rid, ret);
+ }
+ }
+
+ void returnFilesForScan(Long reservationId) {
+
+ final Set<FileRef> filesToDelete = new HashSet<FileRef>();
+
+ synchronized (Tablet.this) {
+ Set<FileRef> absFilePaths = scanFileReservations.remove(reservationId);
+
+ if (absFilePaths == null)
+ throw new IllegalArgumentException("Unknown scan reservation id " + reservationId);
+
+ boolean notify = false;
+ for (FileRef path : absFilePaths) {
+ long refCount = fileScanReferenceCounts.decrement(path, 1);
+ if (refCount == 0) {
+ if (filesToDeleteAfterScan.remove(path))
+ filesToDelete.add(path);
+ notify = true;
+ } else if (refCount < 0)
+ throw new IllegalStateException("Scan ref count for " + path + " is " + refCount);
+ }
+
+ if (notify)
+ Tablet.this.notifyAll();
+ }
+
+ if (filesToDelete.size() > 0) {
+ log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
+ MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
+ }
+ }
+
+ private void removeFilesAfterScan(Set<FileRef> scanFiles) {
+ if (scanFiles.size() == 0)
+ return;
+
+ Set<FileRef> filesToDelete = new HashSet<FileRef>();
+
+ synchronized (Tablet.this) {
+ for (FileRef path : scanFiles) {
+ if (fileScanReferenceCounts.get(path) == 0)
+ filesToDelete.add(path);
+ else
+ filesToDeleteAfterScan.add(path);
+ }
+ }
+
+ if (filesToDelete.size() > 0) {
+ log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
+ MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
+ }
+ }
+
+ private TreeSet<FileRef> waitForScansToFinish(Set<FileRef> pathsToWaitFor, boolean blockNewScans, long maxWaitTime) {
+ long startTime = System.currentTimeMillis();
+ TreeSet<FileRef> inUse = new TreeSet<FileRef>();
+
+ Span waitForScans = Trace.start("waitForScans");
+ try {
+ synchronized (Tablet.this) {
+ if (blockNewScans) {
+ if (reservationsBlocked)
+ throw new IllegalStateException();
+
+ reservationsBlocked = true;
+ }
+
+ for (FileRef path : pathsToWaitFor) {
+ while (fileScanReferenceCounts.get(path) > 0 && System.currentTimeMillis() - startTime < maxWaitTime) {
+ try {
+ Tablet.this.wait(100);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ for (FileRef path : pathsToWaitFor) {
+ if (fileScanReferenceCounts.get(path) > 0)
+ inUse.add(path);
+ }
+
+ if (blockNewScans) {
+ reservationsBlocked = false;
+ Tablet.this.notifyAll();
+ }
+
+ }
+ } finally {
+ waitForScans.stop();
+ }
+ return inUse;
+ }
+
+ public void importMapFiles(long tid, Map<FileRef,DataFileValue> pathsString, boolean setTime) throws IOException {
+
+ String bulkDir = null;
+
+ Map<FileRef,DataFileValue> paths = new HashMap<FileRef,DataFileValue>();
+ for (Entry<FileRef,DataFileValue> entry : pathsString.entrySet())
+ paths.put(entry.getKey(), entry.getValue());
+
+ for (FileRef tpath : paths.keySet()) {
+
+ boolean inTheRightDirectory = false;
+ Path parent = tpath.path().getParent().getParent();
+ for (String tablesDir : ServerConstants.getTablesDirs()) {
+ if (parent.equals(new Path(tablesDir, extent.getTableId().toString()))) {
+ inTheRightDirectory = true;
+ break;
+ }
+ }
+ if (!inTheRightDirectory) {
+ throw new IOException("Data file " + tpath + " not in table dirs");
+ }
+
+ if (bulkDir == null)
+ bulkDir = tpath.path().getParent().toString();
+ else if (!bulkDir.equals(tpath.path().getParent().toString()))
+ throw new IllegalArgumentException("bulk files in different dirs " + bulkDir + " " + tpath);
+
+ }
+
+ if (extent.isRootTablet()) {
+ throw new IllegalArgumentException("Can not import files to root tablet");
+ }
+
+ synchronized (bulkFileImportLock) {
+ Credentials creds = SystemCredentials.get();
+ Connector conn;
+ try {
+ conn = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ // Remove any bulk files we've previously loaded and compacted away
+ List<FileRef> files = MetadataTableUtil.getBulkFilesLoaded(conn, extent, tid);
+
+ for (FileRef file : files)
+ if (paths.keySet().remove(file))
+ log.debug("Ignoring request to re-import a file already imported: " + extent + ": " + file);
+
+ if (paths.size() > 0) {
+ long bulkTime = Long.MIN_VALUE;
+ if (setTime) {
+ for (DataFileValue dfv : paths.values()) {
+ long nextTime = tabletTime.getAndUpdateTime();
+ if (nextTime < bulkTime)
+ throw new IllegalStateException("Time went backwards unexpectedly " + nextTime + " " + bulkTime);
+ bulkTime = nextTime;
+ dfv.setTime(bulkTime);
+ }
+ }
+
+ synchronized (timeLock) {
+ if (bulkTime > persistedTime)
+ persistedTime = bulkTime;
+
+ MetadataTableUtil.updateTabletDataFile(tid, extent, paths, tabletTime.getMetadataValue(persistedTime), creds, tabletServer.getLock());
+ }
+ }
+ }
+
+ synchronized (Tablet.this) {
+ for (Entry<FileRef,DataFileValue> tpath : paths.entrySet()) {
+ if (datafileSizes.containsKey(tpath.getKey())) {
+ log.error("Adding file that is already in set " + tpath.getKey());
+ }
+ datafileSizes.put(tpath.getKey(), tpath.getValue());
+
+ }
+
+ tabletResources.importedMapFiles();
+
+ computeNumEntries();
+ }
+
+ for (Entry<FileRef,DataFileValue> entry : paths.entrySet()) {
+ log.log(TLevel.TABLET_HIST, extent + " import " + entry.getKey() + " " + entry.getValue());
+ }
+ }
+
+ FileRef reserveMergingMinorCompactionFile() {
+ if (mergingMinorCompactionFile != null)
+ throw new IllegalStateException("Tried to reserve merging minor compaction file when already reserved : " + mergingMinorCompactionFile);
+
+ if (extent.isRootTablet())
+ return null;
+
+ int maxFiles = acuTableConf.getMaxFilesPerTablet();
+
+ // when a major compaction is running and we are at max files, write out
+ // one extra file... want to avoid the case where major compaction is
+ // compacting everything except for the largest file, and therefore the
+ // largest file is returned for merging.. the following check mostly
+ // avoids this case, except for the case where major compactions fail or
+ // are canceled
+ if (majorCompactingFiles.size() > 0 && datafileSizes.size() == maxFiles)
+ return null;
+
+ if (datafileSizes.size() >= maxFiles) {
+ // find the smallest file
+
+ long min = Long.MAX_VALUE;
+ FileRef minName = null;
+
+ for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
+ if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey())) {
+ min = entry.getValue().getSize();
+ minName = entry.getKey();
+ }
+ }
+
+ if (minName == null)
+ return null;
+
+ mergingMinorCompactionFile = minName;
+ return minName;
+ }
+
+ return null;
+ }
+
+ void unreserveMergingMinorCompactionFile(FileRef file) {
+ if ((file == null && mergingMinorCompactionFile != null) || (file != null && mergingMinorCompactionFile == null)
+ || (file != null && mergingMinorCompactionFile != null && !file.equals(mergingMinorCompactionFile)))
+ throw new IllegalStateException("Disagreement " + file + " " + mergingMinorCompactionFile);
+
+ mergingMinorCompactionFile = null;
+ }
+
+ void bringMinorCompactionOnline(FileRef tmpDatafile, FileRef newDatafile, FileRef absMergeFile, DataFileValue dfv, CommitSession commitSession, long flushId)
+ throws IOException {
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (extent.isRootTablet()) {
+ try {
+ if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
+ throw new IllegalStateException();
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
+ }
+ }
+
+ // rename before putting in metadata table, so files in metadata table should
+ // always exist
+ do {
+ try {
+ if (dfv.getNumEntries() == 0) {
+ fs.deleteRecursively(tmpDatafile.path());
+ } else {
+ if (fs.exists(newDatafile.path())) {
+ log.warn("Target map file already exist " + newDatafile);
+ fs.deleteRecursively(newDatafile.path());
+ }
+
+ rename(fs, tmpDatafile.path(), newDatafile.path());
+ }
+ break;
+ } catch (IOException ioe) {
+ log.warn("Tablet " + extent + " failed to rename " + newDatafile + " after MinC, will retry in 60 secs...", ioe);
+ UtilWaitThread.sleep(60 * 1000);
+ }
+ } while (true);
+
+ long t1, t2;
+
+ // the code below always assumes merged files are in use by scans... this must be done
+ // because the in memory list of files is not updated until after the metadata table
+ // therefore the file is available to scans until memory is updated, but want to ensure
+ // the file is not available for garbage collection... if memory were updated
+ // before this point (like major compactions do), then the following code could wait
+ // for scans to finish like major compactions do.... used to wait for scans to finish
+ // here, but that was incorrect because a scan could start after waiting but before
+ // memory was updated... assuming the file is always in use by scans leads to
+ // one uneeded metadata update when it was not actually in use
+ Set<FileRef> filesInUseByScans = Collections.emptySet();
+ if (absMergeFile != null)
+ filesInUseByScans = Collections.singleton(absMergeFile);
+
+ // very important to write delete entries outside of log lock, because
+ // this metadata write does not go up... it goes sideways or to itself
+ if (absMergeFile != null)
+ MetadataTableUtil.addDeleteEntries(extent, Collections.singleton(absMergeFile), SystemCredentials.get());
+
+ Set<String> unusedWalLogs = beginClearingUnusedLogs();
+ try {
+ // the order of writing to metadata and walog is important in the face of machine/process failures
+ // need to write to metadata before writing to walog, when things are done in the reverse order
+ // data could be lost... the minor compaction start even should be written before the following metadata
+ // write is made
+
+ synchronized (timeLock) {
+ if (commitSession.getMaxCommittedTime() > persistedTime)
+ persistedTime = commitSession.getMaxCommittedTime();
+
+ String time = tabletTime.getMetadataValue(persistedTime);
+ MasterMetadataUtil.updateTabletDataFile(extent, newDatafile, absMergeFile, dfv, time, SystemCredentials.get(), filesInUseByScans,
+ tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, lastLocation, flushId);
+ }
+
+ } finally {
+ finishClearingUnusedLogs();
+ }
+
+ do {
+ try {
+ // the purpose of making this update use the new commit session, instead of the old one passed in,
+ // is because the new one will reference the logs used by current memory...
+
+ tabletServer.minorCompactionFinished(tabletMemory.getCommitSession(), newDatafile.toString(), commitSession.getWALogSeq() + 2);
+ break;
+ } catch (IOException e) {
+ log.error("Failed to write to write-ahead log " + e.getMessage() + " will retry", e);
+ UtilWaitThread.sleep(1 * 1000);
+ }
+ } while (true);
+
+ synchronized (Tablet.this) {
+ lastLocation = null;
+
+ t1 = System.currentTimeMillis();
+ if (datafileSizes.containsKey(newDatafile)) {
+ log.error("Adding file that is already in set " + newDatafile);
+ }
+
+ if (dfv.getNumEntries() > 0) {
+ datafileSizes.put(newDatafile, dfv);
+ }
+
+ if (absMergeFile != null) {
+ datafileSizes.remove(absMergeFile);
+ }
+
+ unreserveMergingMinorCompactionFile(absMergeFile);
+
+ dataSourceDeletions.incrementAndGet();
+ tabletMemory.finishedMinC();
+
+ lastFlushID = flushId;
+
+ computeNumEntries();
+ t2 = System.currentTimeMillis();
+ }
+
+ // must do this after list of files in memory is updated above
+ removeFilesAfterScan(filesInUseByScans);
+
+ if (absMergeFile != null)
+ log.log(TLevel.TABLET_HIST, extent + " MinC [" + absMergeFile + ",memory] -> " + newDatafile);
+ else
+ log.log(TLevel.TABLET_HIST, extent + " MinC [memory] -> " + newDatafile);
+ log.debug(String.format("MinC finish lock %.2f secs %s", (t2 - t1) / 1000.0, getExtent().toString()));
+ if (dfv.getSize() > acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD)) {
+ log.debug(String.format("Minor Compaction wrote out file larger than split threshold. split threshold = %,d file size = %,d",
+ acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD), dfv.getSize()));
+ }
+
+ }
+
+ public void reserveMajorCompactingFiles(Collection<FileRef> files) {
+ if (majorCompactingFiles.size() != 0)
+ throw new IllegalStateException("Major compacting files not empty " + majorCompactingFiles);
+
+ if (mergingMinorCompactionFile != null && files.contains(mergingMinorCompactionFile))
+ throw new IllegalStateException("Major compaction tried to resrve file in use by minor compaction " + mergingMinorCompactionFile);
+
+ majorCompactingFiles.addAll(files);
+ }
+
+ public void clearMajorCompactingFile() {
+ majorCompactingFiles.clear();
+ }
+
+ void bringMajorCompactionOnline(Set<FileRef> oldDatafiles, FileRef tmpDatafile, FileRef newDatafile, Long compactionId, DataFileValue dfv)
+ throws IOException {
+ long t1, t2;
+
+ if (!extent.isRootTablet()) {
+
+ if (fs.exists(newDatafile.path())) {
+ log.error("Target map file already exist " + newDatafile, new Exception());
+ throw new IllegalStateException("Target map file already exist " + newDatafile);
+ }
+
+ // rename before putting in metadata table, so files in metadata table should
+ // always exist
+ rename(fs, tmpDatafile.path(), newDatafile.path());
+
+ if (dfv.getNumEntries() == 0) {
+ fs.deleteRecursively(newDatafile.path());
+ }
+ }
+
+ TServerInstance lastLocation = null;
+ synchronized (Tablet.this) {
+
+ t1 = System.currentTimeMillis();
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ dataSourceDeletions.incrementAndGet();
+
+ if (extent.isRootTablet()) {
+
+ waitForScansToFinish(oldDatafiles, true, Long.MAX_VALUE);
+
+ try {
+ if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
+ throw new IllegalStateException();
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
+ }
+
+ // mark files as ready for deletion, but
+ // do not delete them until we successfully
+ // rename the compacted map file, in case
+ // the system goes down
+
+ RootFiles.replaceFiles(acuTableConf, fs, location, oldDatafiles, tmpDatafile, newDatafile);
+ }
+
+ // atomically remove old files and add new file
+ for (FileRef oldDatafile : oldDatafiles) {
+ if (!datafileSizes.containsKey(oldDatafile)) {
+ log.error("file does not exist in set " + oldDatafile);
+ }
+ datafileSizes.remove(oldDatafile);
+ majorCompactingFiles.remove(oldDatafile);
+ }
+
+ if (datafileSizes.containsKey(newDatafile)) {
+ log.error("Adding file that is already in set " + newDatafile);
+ }
+
+ if (dfv.getNumEntries() > 0) {
+ datafileSizes.put(newDatafile, dfv);
+ }
+
+ // could be used by a follow on compaction in a multipass compaction
+ majorCompactingFiles.add(newDatafile);
+
+ computeNumEntries();
+
+ lastLocation = Tablet.this.lastLocation;
+ Tablet.this.lastLocation = null;
+
+ if (compactionId != null)
+ lastCompactID = compactionId;
+
+ t2 = System.currentTimeMillis();
+ }
+
+ if (!extent.isRootTablet()) {
+ Set<FileRef> filesInUseByScans = waitForScansToFinish(oldDatafiles, false, 10000);
+ if (filesInUseByScans.size() > 0)
+ log.debug("Adding scan refs to metadata " + extent + " " + filesInUseByScans);
+ MasterMetadataUtil.replaceDatafiles(extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, SystemCredentials.get(),
+ tabletServer.getClientAddressString(), lastLocation, tabletServer.getLock());
+ removeFilesAfterScan(filesInUseByScans);
+ }
+
+ log.debug(String.format("MajC finish lock %.2f secs", (t2 - t1) / 1000.0));
+ log.log(TLevel.TABLET_HIST, extent + " MajC " + oldDatafiles + " --> " + newDatafile);
+ }
+
+ public SortedMap<FileRef,DataFileValue> getDatafileSizes() {
+ synchronized (Tablet.this) {
+ TreeMap<FileRef,DataFileValue> copy = new TreeMap<FileRef,DataFileValue>(datafileSizes);
+ return Collections.unmodifiableSortedMap(copy);
+ }
+ }
+
+ public Set<FileRef> getFiles() {
+ synchronized (Tablet.this) {
+ HashSet<FileRef> files = new HashSet<FileRef>(datafileSizes.keySet());
+ return Collections.unmodifiableSet(files);
+ }
+ }
+
+ }
+
+ public Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
+ throws IOException {
+ this(tabletServer, location, extent, trm, CachedConfiguration.getInstance(), tabletsKeyValues);
+ splitCreationTime = 0;
+ }
+
+ public Tablet(KeyExtent extent, TabletServer tabletServer, TabletResourceManager trm, SplitInfo info) throws IOException {
+ this(tabletServer, new Text(info.dir), extent, trm, CachedConfiguration.getInstance(), info.datafiles, info.time, info.initFlushID, info.initCompactID, info.lastLocation);
+ splitCreationTime = System.currentTimeMillis();
+ }
+
+ /**
+ * Only visibile for testing
+ */
+ @VisibleForTesting
+ protected Tablet(TabletTime tabletTime, String tabletDirectory, int logId, Path location, DatafileManager datafileManager) {
+ this.tabletTime = tabletTime;
+ this.tabletDirectory = tabletDirectory;
+ this.logId = logId;
+ this.location = location;
+ this.datafileManager = datafileManager;
+ }
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+ this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), tabletsKeyValues);
+ }
+
+ static private final List<LogEntry> EMPTY = Collections.emptyList();
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
+ SortedMap<FileRef,DataFileValue> datafiles, String time, long initFlushID, long initCompactID, TServerInstance last) throws IOException {
+ this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), EMPTY, datafiles, time, last, new HashSet<FileRef>(), initFlushID, initCompactID);
+ }
+
+ private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ SortedMap<Key,Value> entries;
+
+ if (extent.isRootTablet()) {
+ return null;
+ } else {
+ entries = new TreeMap<Key,Value>();
+ Text rowName = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
+ entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
+ }
+ }
+ }
+
+ // log.debug("extent : "+extent+" entries : "+entries);
+
+ if (entries.size() == 1)
+ return entries.values().iterator().next().toString();
+ return null;
+ }
+
+ private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+
+ TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
+
+ if (extent.isRootTablet()) { // the meta0 tablet
+ Path location = new Path(MetadataTableUtil.getRootTabletDir());
+
+ // cleanUpFiles() has special handling for delete. files
+ FileStatus[] files = fs.listStatus(location);
+ Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
+ for (String good : goodPaths) {
+ Path path = new Path(good);
+ String filename = path.getName();
+ FileRef ref = new FileRef(location.toString() + "/" + filename, path);
+ DataFileValue dfv = new DataFileValue(0, 0);
+ datafiles.put(ref, dfv);
+ }
+ } else {
+
+ Text rowName = extent.getMetadataEntry();
+
+ String tableId = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
+ ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), tableId, Authorizations.EMPTY);
+
+ // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
+ // reduced batch size to improve performance
+ // changed here after endKeys were implemented from 10 to 1000
+ mdScanner.setBatchSize(1000);
+
+ // leave these in, again, now using endKey for safety
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+
+ mdScanner.setRange(new Range(rowName));
+
+ for (Entry<Key,Value> entry : mdScanner) {
+
+ if (entry.getKey().compareRow(rowName) != 0) {
+ break;
+ }
+
+ FileRef ref = new FileRef(fs, entry.getKey());
+ datafiles.put(ref, new DataFileValue(entry.getValue().get()));
+ }
+ }
+ return datafiles;
+ }
+
+ private static List<LogEntry> lookupLogEntries(KeyExtent ke, SortedMap<Key,Value> tabletsKeyValues) {
+ List<LogEntry> logEntries = new ArrayList<LogEntry>();
+
+ if (ke.isMeta()) {
+ try {
+ logEntries = MetadataTableUtil.getLogEntries(SystemCredentials.get(), ke);
+ } catch (Exception ex) {
+ throw new RuntimeException("Unable to read tablet log entries", ex);
+ }
+ } else {
+ log.debug("Looking at metadata " + tabletsKeyValues);
+ Text row = ke.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row)) {
+ if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
+ logEntries.add(LogEntry.fromKeyValue(key, entry.getValue()));
+ }
+ }
+ }
+ }
+
+ log.debug("got " + logEntries + " for logs for " + ke);
+ return logEntries;
+ }
+
+ private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
+ HashSet<FileRef> scanFiles = new HashSet<FileRef>();
+
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
+ scanFiles.add(new FileRef(fs, key));
+ }
+ }
+
+ return scanFiles;
+ }
+
+ private static long lookupFlushID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ return Long.parseLong(entry.getValue().toString());
+ }
+
+ return -1;
+ }
+
+ private static long lookupCompactID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ return Long.parseLong(entry.getValue().toString());
+ }
+
+ return -1;
+ }
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, VolumeManager fs,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+ this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(), fs,
+ extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent, tabletsKeyValues),
+ lookupScanFiles(extent, tabletsKeyValues, fs), lookupFlushID(extent, tabletsKeyValues), lookupCompactID(extent, tabletsKeyValues));
+ }
+
+ private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
+ return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
+ */
+ private Tablet(final TabletServer tabletServer, final Text location, final KeyExtent extent, final TabletResourceManager trm, final Configuration conf,
+ final VolumeManager fs, final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time,
+ final TServerInstance lastLocation, Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
+
+ TabletFiles tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getLock(), fs, extent, new TabletFiles(location.toString(), rawLogEntries,
+ rawDatafiles));
+
+ Path locationPath;
+
+ if (tabletPaths.dir.contains(":")) {
+ locationPath = new Path(tabletPaths.dir.toString());
+ } else {
+ locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString());
+ }
+
+ final List<LogEntry> logEntries = tabletPaths.logEntries;
+ final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
+
+ this.location = locationPath;
+ this.lastLocation = lastLocation;
+ this.tabletDirectory = tabletPaths.dir;
+ this.conf = conf;
+ this.acuTableConf = tabletServer.getTableConfiguration(extent);
+
+ this.fs = fs;
+ this.extent = extent;
+ this.tabletResources = trm;
+
+ this.lastFlushID = initFlushID;
+ this.lastCompactID = initCompactID;
+
+ if (extent.isRootTablet()) {
+ long rtime = Long.MIN_VALUE;
+ for (FileRef ref : datafiles.keySet()) {
+ Path path = ref.path();
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
+ long maxTime = -1;
+ try {
+
+ while (reader.hasTop()) {
+ maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
+ reader.next();
+ }
+
+ } finally {
+ reader.close();
+ }
+
+ if (maxTime > rtime) {
+ time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
+ rtime = maxTime;
+ }
+ }
+ }
+ if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
+ // recovery... old root tablet has no data, so time doesn't matter:
+ time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+ }
+
+ this.tabletServer = tabletServer;
+ this.logId = tabletServer.createLogId(extent);
+
+ this.timer = new TabletStatsKeeper();
+
+ setupDefaultSecurityLabels(extent);
+
+ tabletMemory = new TabletMemory();
+ tabletTime = TabletTime.getInstance(time);
+ persistedTime = tabletTime.getTime();
+
+ acuTableConf.addObserver(configObserver = new ConfigurationObserver() {
+
+ private void reloadConstraints() {
+ constraintChecker.set(new ConstraintChecker(acuTableConf));
+ }
+
+ @Override
+ public void propertiesChanged() {
+ reloadConstraints();
+
+ try {
+ setupDefaultSecurityLabels(extent);
+ } catch (Exception e) {
+ log.error("Failed to reload default security labels for extent: " + extent.toString());
+ }
+ }
+
+ @Override
+ public void propertyChanged(String prop) {
+ if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
+ reloadConstraints();
+ else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
+ try {
+ log.info("Default security labels changed for extent: " + extent.toString());
+ setupDefaultSecurityLabels(extent);
+ } catch (Exception e) {
+ log.error("Failed to reload default security labels for extent: " + extent.toString());
+ }
+ }
+
+ }
+
+ @Override
+ public void sessionExpired() {
+ log.debug("Session expired, no longer updating per table props...");
+ }
+
+ });
+
+ acuTableConf.getNamespaceConfiguration().addObserver(configObserver);
+
+ // Force a load of any per-table properties
+ configObserver.propertiesChanged();
+
+ tabletResources.setTablet(this, acuTableConf);
+ if (!logEntries.isEmpty()) {
+ log.info("Starting Write-Ahead Log recovery for " + this.extent);
+ // count[0] = entries used on tablet
+ // count[1] = track max time from walog entries wihtout timestamps
+ final long[] count = new long[2];
+ final CommitSession commitSession = tabletMemory.getCommitSession();
+ count[1] = Long.MIN_VALUE;
+ try {
+ Set<String> absPaths = new HashSet<String>();
+ for (FileRef ref : datafiles.keySet())
+ absPaths.add(ref.path().toString());
+
+ tabletServer.recover(this.tabletServer.getFileSystem(), this, logEntries, absPaths, new MutationReceiver() {
+ @Override
+ public void receive(Mutation m) {
+ // LogReader.printMutation(m);
+ Collection<ColumnUpdate> muts = m.getUpdates();
+ for (ColumnUpdate columnUpdate : muts) {
+ if (!columnUpdate.hasTimestamp()) {
+ // if it is not a user set timestamp, it must have been set
+ // by the system
+ count[1] = Math.max(count[1], columnUpdate.getTimestamp());
+ }
+ }
+ tabletMemory.mutate(commitSession, Collections.singletonList(m));
+ count[0]++;
+ }
+ });
+
+ if (count[1] != Long.MIN_VALUE) {
+ tabletTime.useMaxTimeFromWALog(count[1]);
+ }
+ commitSession.updateMaxCommittedTime(tabletTime.getTime());
+
+ if (count[0] == 0) {
+ log.debug("No replayed mutations applied, removing unused entries for " + extent);
+ MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
+ logEntries.clear();
+ }
+
+ } catch (Throwable t) {
+ if (acuTableConf.getBoolean(Property.TABLE_FAILURES_IGNORE)) {
+ log.warn("Error recovering from log files: ", t);
+ } else {
+ throw new RuntimeException(t);
+ }
+ }
+ // make some closed references that represent the recovered logs
+ currentLogs = new HashSet<DfsLogger>();
+ for (LogEntry logEntry : logEntries) {
+ for (String log : logEntry.logSet) {
+ currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), log, logEntry.getColumnQualifier().toString()));
+ }
+ }
+
+ log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries()
+ + " entries created)");
+ }
+
+ String contextName = acuTableConf.get(Property.TABLE_CLASSPATH);
+ if (contextName != null && !contextName.equals("")) {
+ // initialize context classloader, instead of possibly waiting for it to initialize for a scan
+ // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292
+ AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
+ }
+
+ // do this last after tablet is completely setup because it
+ // could cause major compaction to start
+ datafileManager = new DatafileManager(datafiles);
+
+ computeNumEntries();
+
+ datafileManager.removeFilesAfterScan(scanFiles);
+
+ // look for hints of a failure on the previous tablet server
+ if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
+ // look for any temp files hanging around
+ removeOldTemporaryFiles();
+ }
+
+ log.log(TLevel.TABLET_HIST, extent + " opened");
+ }
+
+ private void removeOldTemporaryFiles() {
+ // remove any temporary files created by a previous tablet server
+ try {
+ for (FileStatus tmp : fs.globStatus(new Path(location, "*_tmp"))) {
+ try {
+ log.debug("Removing old temp file " + tmp.getPath());
+ fs.delete(tmp.getPath());
+ } catch (IOException ex) {
+ log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
+ }
+ }
+ } catch (IOException ex) {
+ log.error("Error scanning for old temp files in " + location);
+ }
+ }
+
+ private void setupDefaultSecurityLabels(KeyExtent extent) {
+ if (extent.isMeta()) {
+ defaultSecurityLabel = new byte[0];
+ } else {
+ try {
+ ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
+ this.defaultSecurityLabel = cv.getExpression();
+ } catch (Exception e) {
+ log.error(e, e);
+ this.defaultSecurityLabel = new byte[0];
+ }
+ }
+ }
+
+ public static class KVEntry extends KeyValue {
+ private static final long serialVersionUID = 1L;
+
+ public KVEntry(Key k, Value v) {
+ super(new Key(k), Arrays.copyOf(v.get(), v.get().length));
+ }
+
+ int numBytes() {
+ return getKey().getSize() + getValue().get().length;
+ }
+
+ int estimateMemoryUsed() {
+ return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object
+ }
+ }
+
+ private LookupResult lookup(SortedKeyValueIterator<Key,Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, ArrayList<KVEntry> results,
+ long maxResultsSize) throws IOException {
+
+ LookupResult lookupResult = new LookupResult();
+
+ boolean exceededMemoryUsage = false;
+ boolean tabletClosed = false;
+
+ Set<ByteSequence> cfset = null;
+ if (columnSet.size() > 0)
+ cfset = LocalityGroupUtil.families(columnSet);
+
+ for (Range range : ranges) {
+
+ if (exceededMemoryUsage || tabletClosed) {
+ lookupResult.unfinishedRanges.add(range);
+ continue;
+ }
+
+ int entriesAdded = 0;
+
+ try {
+ if (cfset != null)
+ mmfi.seek(range, cfset, true);
+ else
+ mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
+
+ while (mmfi.hasTop()) {
+ Key key = mmfi.getTopKey();
+
+ KVEntry kve = new KVEntry(key, mmfi.getTopValue());
+ results.add(kve);
+ entriesAdded++;
+ lookupResult.bytesAdded += kve.estimateMemoryUsed();
+ lookupResult.dataSize += kve.numBytes();
+
+ exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
+
+ if (exceededMemoryUsage) {
+ addUnfinishedRange(lookupResult, range, key, false);
+ break;
+ }
+
+ mmfi.next();
+ }
+
+ } catch (TooManyFilesException tmfe) {
+ // treat this as a closed tablet, and let the client retry
+ log.warn("Tablet " + getExtent() + " has too many files, batch lookup can not run");
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } catch (IOException ioe) {
+ if (shutdownInProgress()) {
+ // assume HDFS shutdown hook caused this exception
+ log.debug("IOException while shutdown in progress ", ioe);
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw ioe;
+ }
+ } catch (IterationInterruptedException iie) {
+ if (isClosed()) {
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw iie;
+ }
+ } catch (TabletClosedException tce) {
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ }
+
+ }
+
+ return lookupResult;
+ }
+
+ private void handleTabletClosedDuringScan(ArrayList<KVEntry> results, LookupResult lookupResult, boolean exceededMemoryUsage, Range range, int entriesAdded) {
+ if (exceededMemoryUsage)
+ throw new IllegalStateException("tablet should not exceed memory usage or close, not both");
+
+ if (entriesAdded > 0)
+ addUnfinishedRange(lookupResult, range, results.get(results.size() - 1).getKey(), false);
+ else
+ lookupResult.unfinishedRanges.add(range);
+
+ lookupResult.closed = true;
+ }
+
+ private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) {
+ if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) {
+ Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive());
+ lookupResult.unfinishedRanges.add(nlur);
+ }
+ }
+
+ public static interface KVReceiver {
+ void receive(List<KVEntry> matches) throws IOException;
+ }
+
+ class LookupResult {
+ List<Range> unfinishedRanges = new ArrayList<Range>();
+ long bytesAdded = 0;
+ long dataSize = 0;
+ boolean closed = false;
+ }
+
+ public LookupResult lookup(List<Range> ranges, HashSet<Column> columns, Authorizations authorizations, ArrayList<KVEntry> results, long maxResultSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag) throws IOException {
+
+ if (ranges.size() == 0) {
+ return new LookupResult();
+ }
+
+ ranges = Range.mergeOverlapping(ranges);
+ Collections.sort(ranges);
+
+ Range tabletRange = extent.toDataRange();
+ for (Range range : ranges) {
+ // do a test to see if this range falls within the tablet, if it does not
+ // then clip will throw an exception
+ tabletRange.clip(range);
+ }
+
+ ScanDataSource dataSource = new ScanDataSource(authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
+
+ LookupResult result = null;
+
+ try {
+ SortedKeyValueIterator<Key,Value> iter = new SourceSwitchingIterator(dataSource);
+ result = lookup(iter, ranges, columns, results, maxResultSize);
+ return result;
+ } catch (IOException ioe) {
+ dataSource.close(true);
+ throw ioe;
+ } finally {
+ // code in finally block because always want
+ // to return mapfiles, even when exception is thrown
+ dataSource.close(false);
+
+ synchronized (this) {
+ queryCount += results.size();
+ if (result != null)
+ queryBytes += result.dataSize;
+ }
+ }
+ }
+
+ private Batch nextBatch(SortedKeyValueIterator<Key,Value> iter, Range range, int num, Set<Column> columns) throws IOException {
+
+ // log.info("In nextBatch..");
+
+ List<KVEntry> results = new ArrayList<KVEntry>();
+ Key key = null;
+
+ Value value;
+ long resultSize = 0L;
+ long resultBytes = 0L;
+
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ if (columns.size() == 0) {
+ iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
+ } else {
+ iter.seek(range, LocalityGroupUtil.families(columns), true);
+ }
+
+ Key continueKey = null;
+ boolean skipContinueKey = false;
+
+ boolean endOfTabletReached = false;
+ while (iter.hasTop()) {
+
+ value = iter.getTopValue();
+ key = iter.getTopKey();
+
+ KVEntry kvEntry = new KVEntry(key, value); // copies key and value
+ results.add(kvEntry);
+ resultSize += kvEntry.estimateMemoryUsed();
+ resultBytes += kvEntry.numBytes();
+
+ if (resultSize >= maxResultsSize || results.size() >= num) {
+ continueKey = new Key(key);
+ skipContinueKey = true;
+ break;
+ }
+
+ iter.next();
+ }
+
+ if (iter.hasTop() == false) {
+ endOfTabletReached = true;
+ }
+
+ Batch retBatch = new Batch();
+ retBatch.numBytes = resultBytes;
+
+ if (!endOfTabletReached) {
+ retBatch.continueKey = continueKey;
+ retBatch.skipContinueKey = skipContinueKey;
+ } else {
+ retBatch.continueKey = null;
+ }
+
+ if (endOfTabletReached && results.size() == 0)
+ retBatch.results = null;
+ else
+ retBatch.results = results;
+
+ return retBatch;
+ }
+
+ /**
+ * Determine if a JVM shutdown is in progress.
+ *
+ */
+ private boolean shutdownInProgress() {
+ try {
+ Runtime.getRuntime().removeShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {}
+ }));
+ } catch (IllegalStateException ise) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private class Batch {
+ public boolean skipContinueKey;
+ public List<KVEntry> results;
+ public Key continueKey;
+ public long numBytes;
+ }
+
+ Scanner createScanner(Range range, int num, Set<Column> columns, Authorizations authorizations, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
+ boolean isolated, AtomicBoolean interruptFlag) {
+ // do a test to see if this range falls within the tablet, if it does not
+ // then clip will throw an exception
+ extent.toDataRange().clip(range);
+
+ ScanOptions opts = new ScanOptions(num, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag, isolated);
+ return new Scanner(range, opts);
+ }
+
+ class ScanBatch {
+ boolean more;
+ List<KVEntry> results;
+
+ ScanBatch(List<KVEntry> results, boolean more) {
+ this.results = results;
+ this.more = more;
+ }
+ }
+
+ class Scanner {
+
+ private ScanOptions options;
+ private Range range;
+ private SortedKeyValueIterator<Key,Value> isolatedIter;
+ private ScanDataSource isolatedDataSource;
+ private boolean sawException = false;
+ private boolean scanClosed = false;
+
+ Scanner(Range range, ScanOptions options) {
+ this.range = range;
+ this.options = options;
+ }
+
+ synchronized ScanBatch read() throws IOException, TabletClosedException {
+
+ if (sawException)
+ throw new IllegalStateException("Tried to use scanner after exception occurred.");
+
+ if (scanClosed)
+ throw new IllegalStateException("Tried to use scanner after it was closed.");
+
+ Batch results = null;
+
+ ScanDataSource dataSource;
+
+ if (options.isolated) {
+ if (isolatedDataSource == null)
+ isolatedDataSource = new ScanDataSource(options);
+ dataSource = isolatedDataSource;
+ } else {
+ dataSource = new ScanDataSource(options);
+ }
+
+ try {
+
+ SortedKeyValueIterator<Key,Value> iter;
+
+ if (options.isolated) {
+ if (isolatedIter == null)
+ isolatedIter = new SourceSwitchingIterator(dataSource, true);
+ else
+ isolatedDataSource.fileManager.reattach();
+ iter = isolatedIter;
+ } else {
+ iter = new SourceSwitchingIterator(dataSource, false);
+ }
+
+ results = nextBatch(iter, range, options.num, options.columnSet);
+
+ if (results.results == null) {
+ range = null;
+ return new ScanBatch(new ArrayList<Tablet.KVEntry>(), false);
+ } else if (results.continueKey == null) {
+ return new ScanBatch(results.results, false);
+ } else {
+ range = new Range(results.continueKey, !results.skipContinueKey, range.getEndKey(), range.isEndKeyInclusive());
+ return new ScanBatch(results.results, true);
+ }
+
+ } catch (IterationInterruptedException iie) {
+ sawException = true;
+ if (isClosed())
+ throw new TabletClosedException(iie);
+ else
+ throw iie;
+ } catch (IOException ioe) {
+ if (shutdownInProgress()) {
+ log.debug("IOException while shutdown in progress ", ioe);
+ throw new TabletClosedException(ioe); // assume IOException was caused by execution of HDFS shutdown hook
+ }
+
+ sawException = true;
+ dataSource.close(true);
+ throw ioe;
+ } catch (RuntimeException re) {
+ sawException = true;
+ throw re;
+ } finally {
+ // code in finally block because always want
+ // to return mapfiles, even when exception is thrown
+ if (!options.isolated)
+ dataSource.close(false);
+ else if (dataSource.fileManager != null)
+ dataSource.fileManager.detach();
+
+ synchronized (Tablet.this) {
+ if (results != null && results.results != null) {
+ long more = results.results.size();
+ queryCount += more;
+ queryBytes += results.numBytes;
+ }
+ }
+ }
+ }
+
+ // close and read are synchronized because can not call close on the data source while it is in use
+ // this cloud lead to the case where file iterators that are in use by a thread are returned
+ // to the pool... this would be bad
+ void close() {
+ options.interruptFlag.set(true);
+ synchronized (this) {
+ scanClosed = true;
+ if (isolatedDataSource != null)
+ isolatedDataSource.close(false);
+ }
+ }
+ }
+
+ static class ScanOptions {
+
+ // scan options
+ Authorizations authorizations;
+ byte[] defaultLabels;
+ Set<Column> columnSet;
+ List<IterInfo> ssiList;
+ Map<String,Map<String,String>> ssio;
+ AtomicBoolean interruptFlag;
+ int num;
+ boolean isolated;
+
+ ScanOptions(int num, Authorizations authorizations, byte[] defaultLabels, Set<Column> columnSet, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag, boolean isolated) {
+ this.num = num;
+ this.authorizations = authorizations;
+ this.defaultLabels = defaultLabels;
+ this.columnSet = columnSet;
+ this.ssiList = ssiList;
+ this.ssio = ssio;
+ this.interruptFlag = interruptFlag;
+ this.isolated = isolated;
+ }
+
+ }
+
+ class ScanDataSource implements DataSource {
+
+ // data source state
+ private ScanFileManager fileManager;
+ private SortedKeyValueIterator<Key,Value> iter;
+ private long expectedDeletionCount;
+ private List<MemoryIterator> memIters = null;
+ private long fileReservationId;
+ private AtomicBoolean interruptFlag;
+ private StatsIterator statsIterator;
+
+ ScanOptions options;
+
+ ScanDataSource(Authorizations authorizations, byte[] defaultLabels, HashSet<Column> columnSet, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
+ AtomicBoolean interruptFlag) {
+ expectedDeletionCount = dataSourceDeletions.get();
+ this.options = new ScanOptions(-1, authorizations, defaultLabels, columnSet, ssiList, ssio, interruptFlag, false);
+ this.interruptFlag = interruptFlag;
+ }
+
+ ScanDataSource(ScanOptions options) {
+ expectedDeletionCount = dataSourceDeletions.get();
+ this.options = options;
+ this.interruptFlag = options.interruptFlag;
+ }
+
+ @Override
+ public DataSource getNewDataSource() {
+ if (!isCurrent()) {
+ // log.debug("Switching data sources during a scan");
+ if (memIters != null) {
+ tabletMemory.returnIterators(memIters);
+ memIters = null;
+ datafileManager.returnFilesForScan(fileReservationId);
+ fileReservationId = -1;
+ }
+
+ if (fileManager != null)
+ fileManager.releaseOpenFiles(false);
+
+ expectedDeletionCount = dataSourceDeletions.get();
+ iter = null;
+
+ return this;
+ } else
+ return this;
+ }
+
+ @Override
+ public boolean isCurrent() {
+ return expectedDeletionCount == dataSourceDeletions.get();
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> iterator() throws IOException {
+ if (iter == null)
+ iter = createIterator();
+ return iter;
+ }
+
+ private SortedKeyValueIterator<Key,Value> createIterator() throws IOException {
+
+ Map<FileRef,DataFileValue> files;
+
+ synchronized (Tablet.this) {
+
+ if (memIters != null)
+ throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
+
+ if (Tablet.this.closed)
+ throw new TabletClosedException();
+
+ if (interruptFlag.get())
+ throw new IterationInterruptedException(extent.toString() + " " + interruptFlag.hashCode());
+
+ // only acquire the file manager when we know the tablet is open
+ if (fileManager == null) {
+ fileManager = tabletResources.newScanFileManager();
+ activeScans.add(this);
+ }
+
+ if (fileManager.getNumOpenFiles() != 0)
+ throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
+
+ // set this before trying to get iterators in case
+ // getIterators() throws an exception
+ expectedDeletionCount = dataSourceDeletions.get();
+
+ memIters = tabletMemory.getIterators();
+ Pair<Long,Map<FileRef,DataFileValue>> reservation = datafileManager.reserveFilesForScan();
+ fileReservationId = reservation.getFirst();
+ files = reservation.getSecond();
+ }
+
+ Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, options.isolated);
+
+ List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.size() + memIters.size());
+
+ iters.addAll(mapfiles);
+ iters.addAll(memIters);
+
+ for (SortedKeyValueIterator<Key,Value> skvi : iters)
+ ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
+
+ MultiIterator multiIter = new MultiIterator(iters, extent);
+
+ TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager, files);
+
+ statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, scannedCount);
+
+ DeletingIterator delIter = new DeletingIterator(statsIterator, false);
+
+ ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
+
+ ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, options.columnSet);
+
+ VisibilityFilter visFilter = new VisibilityFilter(colFilter, options.authorizations, options.defaultLabels);
+
+ return iterEnv.getTopLevelIterator(IteratorUtil
+ .loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.ssiList, options.ssio, iterEnv));
+ }
+
+ private void close(boolean sawErrors) {
+
+ if (memIters != null) {
+ tabletMemory.returnIterators(memIters);
+ memIters = null;
+ datafileManager.returnFilesForScan(fileReservationId);
+ fileReservationId = -1;
+ }
+
+ synchronized (Tablet.this) {
+ activeScans.remove(this);
+ if (activeScans.size() == 0)
+ Tablet.this.notifyAll();
+ }
+
+ if (fileManager != null) {
+ fileManager.releaseOpenFiles(sawErrors);
+ fileManager = null;
+ }
+
+ if (statsIterator != null) {
+ statsIterator.report();
+ }
+
+ }
+
+ public void interrupt() {
+ interruptFlag.set(true);
+ }
+
+ @Override
+ public DataSource getDeepCopyDataSource(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private DataFileValue minorCompact(Configuration conf, VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile,
+ boolean hasQueueTime, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
+ boolean failed = false;
+ long start = System.currentTimeMillis();
+ timer.incrementStatusMinor();
+
+ long count = 0;
+
+ try {
+ Span span = Trace.start("write");
+ CompactionStats stats;
+ try {
+ count = memTable.getNumEntries();
+
+ DataFileValue dfv = null;
+ if (mergeFile != null)
+ dfv = datafileManager.getDatafileSizes().get(mergeFile);
+
+ MinorCompactor compactor = new MinorCompactor(conf, fs, memTable, mergeFile, dfv, tmpDatafile, acuTableConf, extent, mincReason);
+ stats = compactor.call();
+ } finally {
+ span.stop();
+ }
+ span = Trace.start("bringOnline");
+ try {
+ datafileManager.bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
+ commitSession, flushId);
+ } finally {
+ span.stop();
+ }
+ return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
+ } catch (Exception E) {
+ failed = true;
+ throw new RuntimeException(E);
+ } catch (Error E) {
+ // Weird errors like "OutOfMemoryError" when trying to create the thread for the compaction
+ failed = true;
+ throw new RuntimeException(E);
+ } finally {
+ try {
+ tabletMemory.finalizeMinC();
+ } catch (Throwable t) {
+ log.error("Failed to free tablet memory", t);
+ }
+
+ if (!failed) {
+ lastMinorCompactionFinishTime = System.currentTimeMillis();
+ }
+ if (tabletServer.mincMetrics.isEnabled())
+ tabletServer.mincMetrics.add(TabletServerMinCMetrics.minc, (lastMinorCompactionFinishTime - start));
+ if (hasQueueTime) {
+ timer.updateTime(Operation.MINOR, queued, start, count, failed);
+ if (tabletServer.mincMetrics.isEnabled())
+ tabletServer.mincMetrics.add(TabletServerMinCMetrics.queue, (start - queued));
+ } else
+ timer.updateTime(Operation.MINOR, start, count, failed);
+ }
+ }
+
+ private class MinorCompactionTask implements Runnable {
+
+ private long queued;
+ private CommitSession commitSession;
+ private DataFileValue stats;
+ private FileRef mergeFile;
+ private long flushId;
+ private MinorCompactionReason mincReason;
+
+ MinorCompactionTask(FileRef mergeFile, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
+ queued = System.currentTimeMillis();
+ minorCompactionWaitingToStart = true;
+ this.commitSession = commitSession;
+ this.mergeFile = mergeFile;
+ this.flushId = flushId;
+ this.mincReason = mincReason;
+ }
+
+ @Override
+ public void run() {
+ minorCompactionWaitingToStart = false;
+ minorCompactionInProgress = true;
+ Span minorCompaction = Trace.on("minorCompaction");
+ try {
+ FileRef newMapfileLocation = getNextMapFilename(mergeFile == null ? "F" : "M");
+ FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp");
+ Span span = Trace.start("waitForCommits");
+ synchronized (Tablet.this) {
+ commitSession.waitForCommitsToFinish();
+ }
+ span.stop();
+ span = Trace.start("start");
+ while (true) {
+ try {
+ // the purpose of the minor compaction start event is to keep track of the filename... in the case
+ // where the metadata table write for the minor compaction finishes and the process dies before
+ // writing the minor compaction finish event, then the start event+filename in metadata table will
+ // prevent recovery of duplicate data... the minor compaction start event could be written at any time
+ // before the metadata write for the minor compaction
+ tabletServer.minorCompactionStarted(commitSession, commitSession.getWALogSeq() + 1, newMapfileLocation.path().toString());
+ break;
+ } catch (IOException e) {
+ log.warn("Failed to write to write ahead log " + e.getMessage(), e);
+ }
+ }
+ span.stop();
+ span = Trace.start("compact");
+ this.stats = minorCompact(conf, fs, tabletMemory.getMinCMemTable(), tmpFileRef, newMapfileLocation, mergeFile, true, queued, commitSession, flushId,
+ mincReason);
+ span.stop();
+
+ if (needsSplit()) {
+ tabletServer.executeSplit(Tablet.this);
+ } else {
+ initiateMajorCompaction(MajorCompactionReason.NORMAL);
+ }
+ } catch (Throwable t) {
+ log.error("Unknown error during minor compaction for extent: " + getExtent(), t);
+ throw new RuntimeException(t);
+ } finally {
+ minorCompactionInProgress = false;
+ minorCompaction.data("extent", extent.toString());
+ minorCompaction.data("numEntries", Long.toString(this.stats.getNumEntries()));
+ minorCompaction.data("size", Long.toString(this.stats.getSize()));
+ minorCompaction.stop();
+ }
+ }
+ }
+
+ private synchronized MinorCompactionTask prepareForMinC(long flushId, MinorCompactionReason mincReason) {
+ CommitSession oldCommitSession = tabletMemory.prepareForMinC();
+ otherLogs = currentLogs;
+ currentLogs = new HashSet<DfsLogger>();
+
+ FileRef mergeFile = null;
+ if (mincReason != MinorCompactionReason.RECOVERY) {
+ mergeFile = datafileManager.reserveMergingMinorCompactionFile();
+ }
+
+ return new MinorCompactionTask(mergeFile, oldCommitSession, flushId, mincReason);
+
+ }
+
+ void flush(long tableFlushID) {
+ boolean updateMetadata = false;
+ boolean initiateMinor = false;
+
+ try {
+
+ synchronized (this) {
+
+ // only want one thing at a time to update flush ID to ensure that metadata table and tablet in memory state are consistent
+ if (updatingFlushID)
+ return;
+
+ if (lastFlushID >= tableFlushID)
+ return;
+
+ if (closing || closed || tabletMemory.memoryReservedForMinC())
+ return;
+
+ if (tabletMemory.getMemTable().getNumEntries() == 0) {
+ lastFlushID = tableFlushID;
+ updatingFlushID = true;
+ updateMetadata = true;
+ } else
+ initiateMinor = true;
+ }
+
+ if (updateMetadata) {
+ Credentials creds = SystemCredentials.get();
+ // if multiple threads were allowed to update this outside of a sync block, then it would be
+ // a race condition
+ MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, tabletServer.getLock());
+ } else if (initiateMinor)
+ initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
+
+ } finally {
+ if (updateMetadata) {
+ synchronized (this) {
+ updatingFlushID = false;
+ this.notifyAll();
+ }
+ }
+ }
+
+ }
+
+ boolean initiateMinorCompaction(MinorCompactionReason mincReason) {
+ if (isClosed()) {
+ // don't bother trying to get flush id if closed... could be closed after this check but that is ok... just trying to cut down on uneeded log messages....
+ return false;
+ }
+
+ // get the flush id before the new memmap is made available for write
+ long flushId;
+ try {
+ flushId = getFlushID();
+ } catch (NoNodeException e) {
+ log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
+ return false;
+ }
+ return initiateMinorCompaction(flushId, mincReason);
+ }
+
+ boolean minorCompactNow(MinorCompactionReason mincReason) {
+ long flushId;
+ try {
+ flushId = getFlushID();
+ } catch (NoNodeException e) {
+ log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
+ return false;
+ }
+ MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
+ if (mct == null)
+ return false;
+ mct.run();
+ return true;
+ }
+
+ boolean initiateMinorCompaction(long flushId, MinorCompactionReason mincReason) {
+ MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
+ if (mct == null)
+ return false;
+ tabletResources.executeMinorCompaction(mct);
+ return true;
+ }
+
+ private MinorCompactionTask createMinorCompactionTask(long flushId, MinorCompactionReason mincReason) {
+ MinorCompactionTask mct;
+ long t1, t2;
+
+ StringBuilder logMessage = null;
+
+ try {
+ synchronized (this) {
+ t1 = System.currentTimeMillis();
+
+ if (closing || closed || majorCompactionWaitingToStart || tabletMemory.memoryReservedForMinC() || tabletMemory.getMemTable().getNumEntries() == 0
+ || updatingFlushID) {
+
+ logMessage = new StringBuilder();
+
+ logMessage.append(extent.toString());
+ logMessage.append(" closing " + closing);
+ logMessage.append(" closed " + closed);
+ logMessage.append(" majorCompactionWaitingToStart " + majorCompactionWaitingToStart);
+ if (tabletMemory != null)
+ logMessage.append(" tabletMemory.memoryReservedForMinC() " + tabletMemory.memoryReservedForMinC());
+ if (tabletMemory != null && tabletMemory.getMemTable() != null)
+ logMessage.append(" tabletMemory.getMemTable().getNumEntries() " + tabletMemory.getMemTable().getNumEntries());
+ logMessage.append(" updatingFlushID " + updatingFlushID);
+
+ return null;
+ }
+
+ mct = prepareForMinC(flushId, mincReason);
+ t2 = System.currentTimeMillis();
+ }
+ } finally {
+ // log outside of sync block
+ if (logMessage != null && log.isDebugEnabled())
+ log.debug(logMessage);
+ }
+
+ log.debug(String.format("MinC initiate lock %.2f secs", (t2 - t1) / 1000.0));
+ return mct;
+ }
+
+ long getFlushID() throws NoNodeException {
+ try {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_FLUSH_ID;
- return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++ return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (NumberFormatException nfe) {
+ throw new RuntimeException(nfe);
+ } catch (KeeperException ke) {
+ if (ke instanceof NoNodeException) {
+ throw (NoNodeException) ke;
+ } else {
+ throw new RuntimeException(ke);
+ }
+ }
+ }
+
+ long getCompactionCancelID() {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ try {
- return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++ return Long.parseLong(new String(ZooReaderWriter.getInstance().get
<TRUNCATED>
[22/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
index c07ed1c,0000000..d77d060
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
@@@ -1,193 -1,0 +1,194 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LargeRowIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setMemory(ServerType.TABLET_SERVER, cfg.getMemory(ServerType.TABLET_SERVER) * 2, MemoryUnit.BYTE);
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "10ms"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private static final int SEED = 42;
+ private static final String REG_TABLE_NAME = "lr";
+ private static final String PRE_SPLIT_TABLE_NAME = "lrps";
+ private static final int NUM_ROWS = 100;
+ private static final int ROW_SIZE = 1 << 17;
+ private static final int NUM_PRE_SPLITS = 9;
+ private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / NUM_PRE_SPLITS;
+
+ private int timeoutFactor = 1;
+
+ @Before
+ public void getTimeoutFactor() {
+ try {
+ timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
+ } catch (NumberFormatException e) {
+ log.warn("Could not parse property value for 'timeout.factor' as integer: " + System.getProperty("timeout.factor"));
+ }
+
+ Assert.assertTrue("Timeout factor must be greater than or equal to 1", timeoutFactor >= 1);
+ }
+
+ @Test
+ public void run() throws Exception {
+ Random r = new Random();
+ byte rowData[] = new byte[ROW_SIZE];
+ r.setSeed(SEED + 1);
+ TreeSet<Text> splitPoints = new TreeSet<Text>();
+ for (int i = 0; i < NUM_PRE_SPLITS; i++) {
+ r.nextBytes(rowData);
+ TestIngest.toPrintableChars(rowData);
+ splitPoints.add(new Text(rowData));
+ }
+ Connector c = getConnector();
+ c.tableOperations().create(REG_TABLE_NAME);
+ c.tableOperations().create(PRE_SPLIT_TABLE_NAME);
+ c.tableOperations().addSplits(PRE_SPLIT_TABLE_NAME, splitPoints);
+ test1(c);
+ test2(c);
+ }
+
+ private void test1(Connector c) throws Exception {
+
+ basicTest(c, REG_TABLE_NAME, 0);
+
+ c.tableOperations().setProperty(REG_TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "" + SPLIT_THRESH);
+
+ UtilWaitThread.sleep(timeoutFactor * 12000);
+ Logger.getLogger(LargeRowIT.class).warn("checking splits");
+ FunctionalTestUtils.checkSplits(c, REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
+
+ verify(c, REG_TABLE_NAME);
+ }
+
+ private void test2(Connector c) throws Exception {
+ basicTest(c, PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
+ }
+
+ private void basicTest(Connector c, String table, int expectedSplits) throws Exception {
+ BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
+
+ Random r = new Random();
+ byte rowData[] = new byte[ROW_SIZE];
+
+ r.setSeed(SEED);
+
+ for (int i = 0; i < NUM_ROWS; i++) {
+
+ r.nextBytes(rowData);
+ TestIngest.toPrintableChars(rowData);
+
+ Mutation mut = new Mutation(new Text(rowData));
- mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ mut.put(new Text(""), new Text(""), new Value(Integer.toString(i).getBytes(UTF_8)));
+ bw.addMutation(mut);
+ }
+
+ bw.close();
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+
+ verify(c, table);
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+
+ c.tableOperations().flush(table, null, null, false);
+
+ // verify while table flush is running
+ verify(c, table);
+
+ // give split time to complete
+ c.tableOperations().flush(table, null, null, true);
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+
+ verify(c, table);
+
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
+ }
+
+ private void verify(Connector c, String table) throws Exception {
+ Random r = new Random();
+ byte rowData[] = new byte[ROW_SIZE];
+
+ r.setSeed(SEED);
+
+ Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
+
+ for (int i = 0; i < NUM_ROWS; i++) {
+
+ r.nextBytes(rowData);
+ TestIngest.toPrintableChars(rowData);
+
+ scanner.setRange(new Range(new Text(rowData)));
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(new Text(rowData))) {
+ throw new Exception("verification failed, unexpected row i =" + i);
+ }
- if (!entry.getValue().equals(Integer.toString(i).getBytes(Constants.UTF8))) {
++ if (!entry.getValue().equals(Integer.toString(i).getBytes(UTF_8))) {
+ throw new Exception("verification failed, unexpected value i =" + i + " value = " + entry.getValue());
+ }
+ count++;
+ }
+
+ if (count != 1) {
+ throw new Exception("verification failed, unexpected count i =" + i + " count=" + count);
+ }
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 3102f59,0000000..f4bee97
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -1,612 -1,0 +1,613 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Random;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.tserver.NativeMap;
+import org.apache.hadoop.io.Text;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class NativeMapIT {
+
+ private Key nk(int r) {
+ return new Key(new Text(String.format("r%09d", r)));
+ }
+
+ private Key nk(int r, int cf, int cq, int cv, int ts, boolean deleted) {
+ Key k = new Key(new Text(String.format("r%09d", r)), new Text(String.format("cf%09d", cf)), new Text(String.format("cq%09d", cq)), new Text(String.format(
+ "cv%09d", cv)), ts);
+
+ k.setDeleted(deleted);
+
+ return k;
+ }
+
+ private Value nv(int v) {
- return new Value(String.format("r%09d", v).getBytes(Constants.UTF8));
++ return new Value(String.format("r%09d", v).getBytes(UTF_8));
+ }
+
+ public static File nativeMapLocation() {
+ File projectDir = new File(System.getProperty("user.dir")).getParentFile();
+ File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION);
+ return nativeMapDir;
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
+ }
+
+ private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {
+ for (int i = start; i <= end; i++) {
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ assertEquals(nk(i), entry.getKey());
+ assertEquals(nv(i + valueOffset), entry.getValue());
+ }
+
+ assertFalse(iter.hasNext());
+ }
+
+ private void insertAndVerify(NativeMap nm, int start, int end, int valueOffset) {
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i + valueOffset));
+ }
+
+ for (int i = start; i <= end; i++) {
+ Value v = nm.get(nk(i));
+ assertNotNull(v);
+ assertEquals(nv(i + valueOffset), v);
+
+ Iterator<Entry<Key,Value>> iter2 = nm.iterator(nk(i));
+ assertTrue(iter2.hasNext());
+ Entry<Key,Value> entry = iter2.next();
+ assertEquals(nk(i), entry.getKey());
+ assertEquals(nv(i + valueOffset), entry.getValue());
+ }
+
+ assertNull(nm.get(nk(start - 1)));
+
+ assertNull(nm.get(nk(end + 1)));
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ verifyIterator(start, end, valueOffset, iter);
+
+ for (int i = start; i <= end; i++) {
+ iter = nm.iterator(nk(i));
+ verifyIterator(i, end, valueOffset, iter);
+
+ // lookup nonexistant key that falls after existing key
+ iter = nm.iterator(nk(i, 1, 1, 1, 1, false));
+ verifyIterator(i + 1, end, valueOffset, iter);
+ }
+
+ assertEquals(end - start + 1, nm.size());
+ }
+
+ private void insertAndVerifyExhaustive(NativeMap nm, int num, int run) {
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = 0; ts < num; ts++) {
+ Key key = nk(i, j, k, l, ts, true);
- Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ nm.put(key, value);
+
+ key = nk(i, j, k, l, ts, false);
- value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ nm.put(key, value);
+ }
+ }
+ }
+ }
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = num - 1; ts >= 0; ts--) {
+ Key key = nk(i, j, k, l, ts, true);
- Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+
+ key = nk(i, j, k, l, ts, false);
- value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ assertTrue(iter.hasNext());
+ entry = iter.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ assertFalse(iter.hasNext());
+
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = 0; ts < num; ts++) {
+ Key key = nk(i, j, k, l, ts, true);
- Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(Constants.UTF8));
++ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ assertEquals(value, nm.get(key));
+
+ Iterator<Entry<Key,Value>> iter2 = nm.iterator(key);
+ assertTrue(iter2.hasNext());
+ Entry<Key,Value> entry = iter2.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+
+ key = nk(i, j, k, l, ts, false);
- value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(Constants.UTF8));
++ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ assertEquals(value, nm.get(key));
+
+ Iterator<Entry<Key,Value>> iter3 = nm.iterator(key);
+ assertTrue(iter3.hasNext());
+ Entry<Key,Value> entry2 = iter3.next();
+ assertEquals(key, entry2.getKey());
+ assertEquals(value, entry2.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ assertEquals(num * num * num * num * num * 2, nm.size());
+ }
+
+ @Test
+ public void test1() {
+ NativeMap nm = new NativeMap();
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ assertFalse(iter.hasNext());
+ nm.delete();
+ }
+
+ @Test
+ public void test2() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+ insertAndVerify(nm, 1, 10, 1);
+ insertAndVerify(nm, 1, 10, 2);
+
+ nm.delete();
+ }
+
+ @Test
+ public void test4() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerifyExhaustive(nm, 3, 0);
+ insertAndVerifyExhaustive(nm, 3, 1);
+
+ nm.delete();
+ }
+
+ @Test
+ public void test5() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ iter.next();
+
+ nm.delete();
+
+ try {
+ nm.put(nk(1), nv(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.get(nk(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.iterator();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.iterator(nk(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.size();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ }
+
+ @Test
+ public void test7() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+
+ nm.delete();
+
+ try {
+ nm.delete();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+ }
+
+ @Test
+ public void test8() {
+ // test verifies that native map sorts keys sharing some common prefix properly
+
+ NativeMap nm = new NativeMap();
+
+ TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
+
+ tm.put(new Key(new Text("fo")), new Value(new byte[] {'0'}));
+ tm.put(new Key(new Text("foo")), new Value(new byte[] {'1'}));
+ tm.put(new Key(new Text("foo1")), new Value(new byte[] {'2'}));
+ tm.put(new Key(new Text("foo2")), new Value(new byte[] {'3'}));
+
+ for (Entry<Key,Value> entry : tm.entrySet()) {
+ nm.put(entry.getKey(), entry.getValue());
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ for (Entry<Key,Value> entry : tm.entrySet()) {
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry2 = iter.next();
+
+ assertEquals(entry.getKey(), entry2.getKey());
+ assertEquals(entry.getValue(), entry2.getValue());
+ }
+
+ assertFalse(iter.hasNext());
+
+ nm.delete();
+ }
+
+ @Test
+ public void test9() {
+ NativeMap nm = new NativeMap();
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (NoSuchElementException e) {
+
+ }
+
+ insertAndVerify(nm, 1, 1, 0);
+
+ iter = nm.iterator();
+ iter.next();
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (NoSuchElementException e) {
+
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void test10() {
+ int start = 1;
+ int end = 10000;
+
+ NativeMap nm = new NativeMap();
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i));
+ }
+
+ long mem1 = nm.getMemoryUsed();
+
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i));
+ }
+
+ long mem2 = nm.getMemoryUsed();
+
+ if (mem1 != mem2) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem2);
+ }
+
+ for (int i = start; i <= end; i++) {
+ nm.put(nk(i), nv(i));
+ }
+
+ long mem3 = nm.getMemoryUsed();
+
+ if (mem1 != mem3) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem3);
+ }
+
+ byte bigrow[] = new byte[1000000];
+ byte bigvalue[] = new byte[bigrow.length];
+
+ for (int i = 0; i < bigrow.length; i++) {
+ bigrow[i] = (byte) (0xff & (i % 256));
+ bigvalue[i] = bigrow[i];
+ }
+
+ nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
+
+ long mem4 = nm.getMemoryUsed();
+
+ Value val = nm.get(new Key(new Text(bigrow)));
+ if (val == null || !val.equals(new Value(bigvalue))) {
+ throw new RuntimeException("Did not get expected big value");
+ }
+
+ nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
+
+ long mem5 = nm.getMemoryUsed();
+
+ if (mem4 != mem5) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem4 + " " + mem5);
+ }
+
+ val = nm.get(new Key(new Text(bigrow)));
+ if (val == null || !val.equals(new Value(bigvalue))) {
+ throw new RuntimeException("Did not get expected big value");
+ }
+
+ nm.delete();
+ }
+
+ // random length random field
+ private static byte[] rlrf(Random r, int maxLen) {
+ int len = r.nextInt(maxLen);
+
+ byte f[] = new byte[len];
+ r.nextBytes(f);
+
+ return f;
+ }
+
+ @Test
+ public void test11() {
+ NativeMap nm = new NativeMap();
+
+ // insert things with varying field sizes and value sizes
+
+ // generate random data
+ Random r = new Random(75);
+
+ ArrayList<Pair<Key,Value>> testData = new ArrayList<Pair<Key,Value>>();
+
+ for (int i = 0; i < 100000; i++) {
+
+ Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false);
+ Value v = new Value(rlrf(r, 511));
+
+ testData.add(new Pair<Key,Value>(k, v));
+ }
+
+ // insert unsorted data
+ for (Pair<Key,Value> pair : testData) {
+ nm.put(pair.getFirst(), pair.getSecond());
+ }
+
+ for (int i = 0; i < 2; i++) {
+
+ // sort data
+ Collections.sort(testData, new Comparator<Pair<Key,Value>>() {
+ @Override
+ public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
+ return o1.getFirst().compareTo(o2.getFirst());
+ }
+ });
+
+ // verify
+ Iterator<Entry<Key,Value>> iter1 = nm.iterator();
+ Iterator<Pair<Key,Value>> iter2 = testData.iterator();
+
+ while (iter1.hasNext() && iter2.hasNext()) {
+ Entry<Key,Value> e = iter1.next();
+ Pair<Key,Value> p = iter2.next();
+
+ if (!e.getKey().equals(p.getFirst()))
+ throw new RuntimeException("Keys not equal");
+
+ if (!e.getValue().equals(p.getSecond()))
+ throw new RuntimeException("Values not equal");
+ }
+
+ if (iter1.hasNext())
+ throw new RuntimeException("Not all of native map consumed");
+
+ if (iter2.hasNext())
+ throw new RuntimeException("Not all of test data consumed");
+
+ System.out.println("test 11 nm mem " + nm.getMemoryUsed());
+
+ // insert data again w/ different value
+ Collections.shuffle(testData, r);
+ // insert unsorted data
+ for (Pair<Key,Value> pair : testData) {
+ pair.getSecond().set(rlrf(r, 511));
+ nm.put(pair.getFirst(), pair.getSecond());
+ }
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void testBinary() {
+ NativeMap nm = new NativeMap();
+
+ byte emptyBytes[] = new byte[0];
+
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ nm.put(k, v);
+ }
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+
+ assertEquals(k, entry.getKey());
+ assertEquals(v, entry.getValue());
+
+ }
+ }
+
+ assertFalse(iter.hasNext());
+
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ Value v2 = nm.get(k);
+
+ assertEquals(v, v2);
+ }
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void testEmpty() {
+ NativeMap nm = new NativeMap();
+
+ assertTrue(nm.size() == 0);
+ assertTrue(nm.getMemoryUsed() == 0);
+
+ nm.delete();
+ }
+
+ @Test
+ public void testConcurrentIter() throws IOException {
+ NativeMap nm = new NativeMap();
+
+ nm.put(nk(0), nv(0));
+ nm.put(nk(1), nv(1));
+ nm.put(nk(3), nv(3));
+
+ SortedKeyValueIterator<Key,Value> iter = nm.skvIterator();
+
+ // modify map after iter created
+ nm.put(nk(2), nv(2));
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(0));
+ iter.next();
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(1));
+ iter.next();
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(2));
+ iter.next();
+
+ assertTrue(iter.hasTop());
+ assertEquals(iter.getTopKey(), nk(3));
+ iter.next();
+
+ assertFalse(iter.hasTop());
+
+ nm.delete();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index 7e43812,0000000..189a55c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@@ -1,132 -1,0 +1,133 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ScanIteratorIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 30;
+ }
+
+ @Test
+ public void run() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 0; i < 1000; i++) {
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(Constants.UTF8)));
- m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value(Integer.toString(1000 - i).getBytes(UTF_8)));
++ m.put(new Text("cf1"), new Text("cq2"), new Value(Integer.toString(i - 1000).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ Scanner scanner = c.createScanner(tableName, new Authorizations());
+
+ setupIter(scanner);
+ verify(scanner, 1, 999);
+
+ BatchScanner bscanner = c.createBatchScanner(tableName, new Authorizations(), 3);
+ bscanner.setRanges(Collections.singleton(new Range((Key) null, null)));
+
+ setupIter(bscanner);
+ verify(bscanner, 1, 999);
+
+ ArrayList<Range> ranges = new ArrayList<Range>();
+ ranges.add(new Range(new Text(String.format("%06d", 1))));
+ ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16))));
+ ranges.add(new Range(new Text(String.format("%06d", 20))));
+ ranges.add(new Range(new Text(String.format("%06d", 23))));
+ ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61))));
+ ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504))));
+ ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000))));
+
+ HashSet<Integer> got = new HashSet<Integer>();
+ HashSet<Integer> expected = new HashSet<Integer>();
+ for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) {
+ expected.add(i);
+ }
+
+ bscanner.setRanges(ranges);
+
+ for (Entry<Key,Value> entry : bscanner) {
+ got.add(Integer.parseInt(entry.getKey().getRow().toString()));
+ }
+
+ System.out.println("got : " + got);
+
+ if (!got.equals(expected)) {
+ throw new Exception(got + " != " + expected);
+ }
+
+ bscanner.close();
+
+ }
+
+ private void verify(Iterable<Entry<Key,Value>> scanner, int start, int finish) throws Exception {
+
+ int expected = start;
+ for (Entry<Key,Value> entry : scanner) {
+ if (Integer.parseInt(entry.getKey().getRow().toString()) != expected) {
+ throw new Exception("Saw unexpexted " + entry.getKey().getRow() + " " + expected);
+ }
+
+ if (entry.getKey().getColumnQualifier().toString().equals("cq2")) {
+ expected += 2;
+ }
+ }
+
+ if (expected != finish + 2) {
+ throw new Exception("Ended at " + expected + " not " + (finish + 2));
+ }
+ }
+
+ private void setupIter(ScannerBase scanner) throws Exception {
+ IteratorSetting dropMod = new IteratorSetting(50, "dropMod", "org.apache.accumulo.test.functional.DropModIter");
+ dropMod.addOption("mod", "2");
+ dropMod.addOption("drop", "0");
+ scanner.addScanIterator(dropMod);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
index ba03d00,0000000..90b881c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
@@@ -1,242 -1,0 +1,243 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ScanRangeIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ private static final int TS_LIMIT = 1;
+ private static final int CQ_LIMIT = 5;
+ private static final int CF_LIMIT = 5;
+ private static final int ROW_LIMIT = 100;
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = getUniqueNames(2);
+ String table1 = tableNames[0];
+ c.tableOperations().create(table1);
+ String table2 = tableNames[1];
+ c.tableOperations().create(table2);
+ TreeSet<Text> splitRows = new TreeSet<Text>();
+ int splits = 3;
+ for (int i = (ROW_LIMIT / splits); i < ROW_LIMIT; i += (ROW_LIMIT / splits))
+ splitRows.add(createRow(i));
+ c.tableOperations().addSplits(table2, splitRows);
+
+ insertData(c, table1);
+ scanTable(c, table1);
+
+ insertData(c, table2);
+ scanTable(c, table2);
+ }
+
+ private void scanTable(Connector c, String table) throws Exception {
+ scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(1, 0, 0, 0));
+
+ scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+
+ scanRange(c, table, null, null);
+
+ for (int i = 0; i < ROW_LIMIT; i += (ROW_LIMIT / 3)) {
+ for (int j = 0; j < CF_LIMIT; j += (CF_LIMIT / 2)) {
+ for (int k = 1; k < CQ_LIMIT; k += (CQ_LIMIT / 2)) {
+ scanRange(c, table, null, new IntKey(i, j, k, 0));
+ scanRange(c, table, new IntKey(0, 0, 0, 0), new IntKey(i, j, k, 0));
+
+ scanRange(c, table, new IntKey(i, j, k, 0), new IntKey(ROW_LIMIT - 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+
+ scanRange(c, table, new IntKey(i, j, k, 0), null);
+
+ }
+ }
+ }
+
+ for (int i = 0; i < ROW_LIMIT; i++) {
+ scanRange(c, table, new IntKey(i, 0, 0, 0), new IntKey(i, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+
+ if (i > 0 && i < ROW_LIMIT - 1) {
+ scanRange(c, table, new IntKey(i - 1, 0, 0, 0), new IntKey(i + 1, CF_LIMIT - 1, CQ_LIMIT - 1, 0));
+ }
+ }
+
+ }
+
+ private static class IntKey {
+ private int row;
+ private int cf;
+ private int cq;
+ private long ts;
+
+ IntKey(IntKey ik) {
+ this.row = ik.row;
+ this.cf = ik.cf;
+ this.cq = ik.cq;
+ this.ts = ik.ts;
+ }
+
+ IntKey(int row, int cf, int cq, long ts) {
+ this.row = row;
+ this.cf = cf;
+ this.cq = cq;
+ this.ts = ts;
+ }
+
+ Key createKey() {
+ Text trow = createRow(row);
+ Text tcf = createCF(cf);
+ Text tcq = createCQ(cq);
+
+ return new Key(trow, tcf, tcq, ts);
+ }
+
+ IntKey increment() {
+
+ IntKey ik = new IntKey(this);
+
+ ik.ts++;
+ if (ik.ts >= TS_LIMIT) {
+ ik.ts = 0;
+ ik.cq++;
+ if (ik.cq >= CQ_LIMIT) {
+ ik.cq = 0;
+ ik.cf++;
+ if (ik.cf >= CF_LIMIT) {
+ ik.cf = 0;
+ ik.row++;
+ }
+ }
+ }
+
+ return ik;
+ }
+
+ }
+
+ private void scanRange(Connector c, String table, IntKey ik1, IntKey ik2) throws Exception {
+ scanRange(c, table, ik1, false, ik2, false);
+ scanRange(c, table, ik1, false, ik2, true);
+ scanRange(c, table, ik1, true, ik2, false);
+ scanRange(c, table, ik1, true, ik2, true);
+ }
+
+ private void scanRange(Connector c, String table, IntKey ik1, boolean inclusive1, IntKey ik2, boolean inclusive2) throws Exception {
+ Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
+
+ Key key1 = null;
+ Key key2 = null;
+
+ IntKey expectedIntKey;
+ IntKey expectedEndIntKey;
+
+ if (ik1 != null) {
+ key1 = ik1.createKey();
+ expectedIntKey = ik1;
+
+ if (!inclusive1) {
+ expectedIntKey = expectedIntKey.increment();
+ }
+ } else {
+ expectedIntKey = new IntKey(0, 0, 0, 0);
+ }
+
+ if (ik2 != null) {
+ key2 = ik2.createKey();
+ expectedEndIntKey = ik2;
+
+ if (inclusive2) {
+ expectedEndIntKey = expectedEndIntKey.increment();
+ }
+ } else {
+ expectedEndIntKey = new IntKey(ROW_LIMIT, 0, 0, 0);
+ }
+
+ Range range = new Range(key1, inclusive1, key2, inclusive2);
+
+ scanner.setRange(range);
+
+ for (Entry<Key,Value> entry : scanner) {
+
+ Key expectedKey = expectedIntKey.createKey();
+ if (!expectedKey.equals(entry.getKey())) {
+ throw new Exception(" " + expectedKey + " != " + entry.getKey());
+ }
+
+ expectedIntKey = expectedIntKey.increment();
+ }
+
+ if (!expectedIntKey.createKey().equals(expectedEndIntKey.createKey())) {
+ throw new Exception(" " + expectedIntKey.createKey() + " != " + expectedEndIntKey.createKey());
+ }
+ }
+
+ private static Text createCF(int cf) {
+ Text tcf = new Text(String.format("cf_%03d", cf));
+ return tcf;
+ }
+
+ private static Text createCQ(int cf) {
+ Text tcf = new Text(String.format("cq_%03d", cf));
+ return tcf;
+ }
+
+ private static Text createRow(int row) {
+ Text trow = new Text(String.format("r_%06d", row));
+ return trow;
+ }
+
+ private void insertData(Connector c, String table) throws Exception {
+
+ BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
+
+ for (int i = 0; i < ROW_LIMIT; i++) {
+ Mutation m = new Mutation(createRow(i));
+
+ for (int j = 0; j < CF_LIMIT; j++) {
+ for (int k = 0; k < CQ_LIMIT; k++) {
+ for (int t = 0; t < TS_LIMIT; t++) {
- m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(Constants.UTF8)));
++ m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(UTF_8)));
+ }
+ }
+ }
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
index ed2f0ba,0000000..3547b68
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
@@@ -1,111 -1,0 +1,112 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ScanSessionTimeOutIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_SESSION_MAXIDLE.getKey(), "3"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("abc");
+
+ BatchWriter bw = c.createBatchWriter("abc", new BatchWriterConfig());
+
+ for (int i = 0; i < 100000; i++) {
+ Mutation m = new Mutation(new Text(String.format("%08d", i)));
+ for (int j = 0; j < 3; j++)
- m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ Scanner scanner = c.createScanner("abc", new Authorizations());
+ scanner.setBatchSize(1000);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ verify(iter, 0, 200);
+
+ // sleep three times the session timeout
+ UtilWaitThread.sleep(9000);
+
+ verify(iter, 200, 100000);
+
+ }
+
+ private void verify(Iterator<Entry<Key,Value>> iter, int start, int stop) throws Exception {
+ for (int i = start; i < stop; i++) {
+
+ Text er = new Text(String.format("%08d", i));
+
+ for (int j = 0; j < 3; j++) {
+ Entry<Key,Value> entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(er)) {
+ throw new Exception("row " + entry.getKey().getRow() + " != " + er);
+ }
+
+ if (!entry.getKey().getColumnFamily().equals(new Text("cf1"))) {
+ throw new Exception("cf " + entry.getKey().getColumnFamily() + " != cf1");
+ }
+
+ if (!entry.getKey().getColumnQualifier().equals(new Text("cq" + j))) {
+ throw new Exception("cq " + entry.getKey().getColumnQualifier() + " != cq" + j);
+ }
+
+ if (!entry.getValue().toString().equals("" + i + "_" + j)) {
+ throw new Exception("value " + entry.getValue() + " != " + i + "_" + j);
+ }
+
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index ea92949,0000000..43d3a86
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@@ -1,272 -1,0 +1,273 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.util.FileUtil;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class SplitRecoveryIT extends ConfigurableMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ private KeyExtent nke(String table, String endRow, String prevEndRow) {
+ return new KeyExtent(new Text(table), endRow == null ? null : new Text(endRow), prevEndRow == null ? null : new Text(prevEndRow));
+ }
+
+ private void run() throws Exception {
+ String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/testLock";
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.OVERWRITE);
+ ZooLock zl = new ZooLock(zPath);
+ boolean gotLock = zl.tryLock(new LockWatcher() {
+
+ @Override
+ public void lostLock(LockLossReason reason) {
+ System.exit(-1);
+
+ }
+
+ @Override
+ public void unableToMonitorLockNode(Throwable e) {
+ System.exit(-1);
+ }
- }, "foo".getBytes(Constants.UTF8));
++ }, "foo".getBytes(UTF_8));
+
+ if (!gotLock) {
+ System.err.println("Failed to get lock " + zPath);
+ }
+
+ // run test for a table with one tablet
+ runSplitRecoveryTest(0, "sp", 0, zl, nke("foo0", null, null));
+ runSplitRecoveryTest(1, "sp", 0, zl, nke("foo1", null, null));
+
+ // run test for tables with two tablets, run test on first and last tablet
+ runSplitRecoveryTest(0, "k", 0, zl, nke("foo2", "m", null), nke("foo2", null, "m"));
+ runSplitRecoveryTest(1, "k", 0, zl, nke("foo3", "m", null), nke("foo3", null, "m"));
+ runSplitRecoveryTest(0, "o", 1, zl, nke("foo4", "m", null), nke("foo4", null, "m"));
+ runSplitRecoveryTest(1, "o", 1, zl, nke("foo5", "m", null), nke("foo5", null, "m"));
+
+ // run test for table w/ three tablets, run test on middle tablet
+ runSplitRecoveryTest(0, "o", 1, zl, nke("foo6", "m", null), nke("foo6", "r", "m"), nke("foo6", null, "r"));
+ runSplitRecoveryTest(1, "o", 1, zl, nke("foo7", "m", null), nke("foo7", "r", "m"), nke("foo7", null, "r"));
+
+ // run test for table w/ three tablets, run test on first
+ runSplitRecoveryTest(0, "g", 0, zl, nke("foo8", "m", null), nke("foo8", "r", "m"), nke("foo8", null, "r"));
+ runSplitRecoveryTest(1, "g", 0, zl, nke("foo9", "m", null), nke("foo9", "r", "m"), nke("foo9", null, "r"));
+
+ // run test for table w/ three tablets, run test on last tablet
+ runSplitRecoveryTest(0, "w", 2, zl, nke("fooa", "m", null), nke("fooa", "r", "m"), nke("fooa", null, "r"));
+ runSplitRecoveryTest(1, "w", 2, zl, nke("foob", "m", null), nke("foob", "r", "m"), nke("foob", null, "r"));
+ }
+
+ private void runSplitRecoveryTest(int failPoint, String mr, int extentToSplit, ZooLock zl, KeyExtent... extents) throws Exception {
+
+ Text midRow = new Text(mr);
+
+ SortedMap<FileRef,DataFileValue> splitMapFiles = null;
+
+ for (int i = 0; i < extents.length; i++) {
+ KeyExtent extent = extents[i];
+
+ String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString() + "/dir_" + i;
+ MetadataTableUtil.addTablet(extent, tdir, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID, zl);
+ SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>();
+ mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i));
+
+ if (i == extentToSplit) {
+ splitMapFiles = mapFiles;
+ }
+ int tid = 0;
+ TransactionWatcher.ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
+ MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles, "L0", SystemCredentials.get(), zl);
+ }
+
+ KeyExtent extent = extents[extentToSplit];
+
+ KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
+ KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
+
+ splitPartiallyAndRecover(extent, high, low, .4, splitMapFiles, midRow, "localhost:1234", failPoint, zl);
+ }
+
+ private void splitPartiallyAndRecover(KeyExtent extent, KeyExtent high, KeyExtent low, double splitRatio, SortedMap<FileRef,DataFileValue> mapFiles,
+ Text midRow, String location, int steps, ZooLock zl) throws Exception {
+
+ SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>();
+ SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
+ List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
+
+ MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, new HashMap<FileRef,FileUtil.FileInfo>(), mapFiles, lowDatafileSizes,
+ highDatafileSizes, highDatafilesToRemove);
+
+ MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, SystemCredentials.get(), zl);
+ TServerInstance instance = new TServerInstance(location, zl.getSessionId());
+ Writer writer = new Writer(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID);
+ Assignment assignment = new Assignment(high, instance);
+ Mutation m = new Mutation(assignment.tablet.getMetadataEntry());
+ m.put(TabletsSection.FutureLocationColumnFamily.NAME, assignment.server.asColumnQualifier(), assignment.server.asMutationValue());
+ writer.update(m);
+
+ if (steps >= 1) {
+ Map<FileRef,Long> bulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), extent);
+ MasterMetadataUtil.addNewTablet(low, "/lowDir", instance, lowDatafileSizes, bulkFiles, SystemCredentials.get(), TabletTime.LOGICAL_TIME_ID + "0", -1l,
+ -1l, zl);
+ }
+ if (steps >= 2)
+ MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, SystemCredentials.get(), zl);
+
+ TabletServer.verifyTabletInformation(high, instance, null, "127.0.0.1:0", zl);
+
+ if (steps >= 1) {
+ ensureTabletHasNoUnexpectedMetadataEntries(low, lowDatafileSizes);
+ ensureTabletHasNoUnexpectedMetadataEntries(high, highDatafileSizes);
+
+ Map<FileRef,Long> lowBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), low);
+ Map<FileRef,Long> highBulkFiles = MetadataTableUtil.getBulkFilesLoaded(SystemCredentials.get(), high);
+
+ if (!lowBulkFiles.equals(highBulkFiles)) {
+ throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
+ }
+
+ if (lowBulkFiles.size() == 0) {
+ throw new Exception(" no bulk files " + low);
+ }
+ } else {
+ ensureTabletHasNoUnexpectedMetadataEntries(extent, mapFiles);
+ }
+ }
+
+ private void ensureTabletHasNoUnexpectedMetadataEntries(KeyExtent extent, SortedMap<FileRef,DataFileValue> expectedMapFiles) throws Exception {
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), MetadataTable.ID, Authorizations.EMPTY);
+ scanner.setRange(extent.toMetadataRange());
+
+ HashSet<ColumnFQ> expectedColumns = new HashSet<ColumnFQ>();
+ expectedColumns.add(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN);
+ expectedColumns.add(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
+ expectedColumns.add(TabletsSection.ServerColumnFamily.TIME_COLUMN);
+ expectedColumns.add(TabletsSection.ServerColumnFamily.LOCK_COLUMN);
+
+ HashSet<Text> expectedColumnFamilies = new HashSet<Text>();
+ expectedColumnFamilies.add(DataFileColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.FutureLocationColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.CurrentLocationColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.LastLocationColumnFamily.NAME);
+ expectedColumnFamilies.add(TabletsSection.BulkFileColumnFamily.NAME);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ while (iter.hasNext()) {
+ Key key = iter.next().getKey();
+
+ if (!key.getRow().equals(extent.getMetadataEntry())) {
+ throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
+ }
+
+ if (expectedColumnFamilies.contains(key.getColumnFamily())) {
+ continue;
+ }
+
+ if (expectedColumns.remove(new ColumnFQ(key))) {
+ continue;
+ }
+
+ throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
+ }
+ System.out.println("expectedColumns " + expectedColumns);
+ if (expectedColumns.size() > 1 || (expectedColumns.size() == 1)) {
+ throw new Exception("Not all expected columns seen " + extent + " " + expectedColumns);
+ }
+
+ SortedMap<FileRef,DataFileValue> fixedMapFiles = MetadataTableUtil.getDataFileSizes(extent, SystemCredentials.get());
+ verifySame(expectedMapFiles, fixedMapFiles);
+ }
+
+ private void verifySame(SortedMap<FileRef,DataFileValue> datafileSizes, SortedMap<FileRef,DataFileValue> fixedDatafileSizes) throws Exception {
+
+ if (!datafileSizes.keySet().containsAll(fixedDatafileSizes.keySet()) || !fixedDatafileSizes.keySet().containsAll(datafileSizes.keySet())) {
+ throw new Exception("Key sets not the same " + datafileSizes.keySet() + " != " + fixedDatafileSizes.keySet());
+ }
+
+ for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
+ DataFileValue dfv = entry.getValue();
+ DataFileValue otherDfv = fixedDatafileSizes.get(entry.getKey());
+
+ if (!dfv.equals(otherDfv)) {
+ throw new Exception(entry.getKey() + " dfv not equal " + dfv + " " + otherDfv);
+ }
+ }
+ }
+
+
+ public static void main(String[] args) throws Exception {
+ new SplitRecoveryIT().run();
+ }
+
+ @Test
+ public void test() throws Exception {
+ assertEquals(0, exec(SplitRecoveryIT.class).waitFor());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
index b74399d,0000000..fccc79f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
@@@ -1,101 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class TabletIT extends ConfigurableMacIT {
+
+ private static final int N = 1000;
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ Map<String,String> siteConfig = new HashMap<String,String>();
+ siteConfig.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "200");
+ siteConfig.put(Property.TSERV_MAXMEM.getKey(), "128M");
+ cfg.setDefaultMemory(256, MemoryUnit.MEGABYTE);
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void createTableTest() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ createTableTest(tableName, false);
+ createTableTest(tableName, true);
+ }
+
+ public void createTableTest(String tableName, boolean readOnly) throws Exception {
+ // create the test table within accumulo
+ Connector connector = getConnector();
+
+ if (!readOnly) {
+ TreeSet<Text> keys = new TreeSet<Text>();
+ for (int i = N / 100; i < N; i += N / 100) {
+ keys.add(new Text(String.format("%05d", i)));
+ }
+
+ // presplit
+ connector.tableOperations().create(tableName);
+ connector.tableOperations().addSplits(tableName, keys);
+ BatchWriter b = connector.createBatchWriter(tableName, new BatchWriterConfig());
+
+ // populate
+ for (int i = 0; i < N; i++) {
+ Mutation m = new Mutation(new Text(String.format("%05d", i)));
- m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
++ m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
+ b.addMutation(m);
+ }
+ b.close();
+ }
+
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ int count = 0;
+ for (Entry<Key,Value> elt : scanner) {
+ String expected = String.format("%05d", count);
+ assert (elt.getKey().getRow().toString().equals(expected));
+ count++;
+ }
+ assertEquals(N, count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
index 1dd0bf1,0000000..f2460cf
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@@ -1,303 -1,0 +1,304 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ByteArraySet;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class VisibilityIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = getUniqueNames(2);
+ String table = tableNames[0];
+ c.tableOperations().create(table);
+ String table2 = tableNames[1];
+ c.tableOperations().create(table2);
+ c.tableOperations().setProperty(table2, Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "DEFLABEL");
+
+ insertData(c, table);
+ queryData(c, table);
+ deleteData(c, table);
+
+ insertDefaultData(c, table2);
+ queryDefaultData(c, table2);
+
+ }
+
+ private static SortedSet<String> nss(String... labels) {
+ TreeSet<String> ts = new TreeSet<String>();
+
+ for (String s : labels) {
+ ts.add(s);
+ }
+
+ return ts;
+ }
+
+ private void mput(Mutation m, String cf, String cq, String cv, String val) {
- ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
- m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(Constants.UTF8)));
++ ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
++ m.put(new Text(cf), new Text(cq), le, new Value(val.getBytes(UTF_8)));
+ }
+
+ private void mputDelete(Mutation m, String cf, String cq, String cv) {
- ColumnVisibility le = new ColumnVisibility(cv.getBytes(Constants.UTF8));
++ ColumnVisibility le = new ColumnVisibility(cv.getBytes(UTF_8));
+ m.putDelete(new Text(cf), new Text(cq), le);
+ }
+
+ private void insertData(Connector c, String tableName) throws Exception {
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m1 = new Mutation(new Text("row1"));
+
+ mput(m1, "cf1", "cq1", "", "v1");
+ mput(m1, "cf1", "cq1", "A", "v2");
+ mput(m1, "cf1", "cq1", "B", "v3");
+ mput(m1, "cf1", "cq1", "A&B", "v4");
+ mput(m1, "cf1", "cq1", "A&(L|M)", "v5");
+ mput(m1, "cf1", "cq1", "B&(L|M)", "v6");
+ mput(m1, "cf1", "cq1", "A&B&(L|M)", "v7");
+ mput(m1, "cf1", "cq1", "A&B&(L)", "v8");
+ mput(m1, "cf1", "cq1", "A&FOO", "v9");
+ mput(m1, "cf1", "cq1", "A&FOO&(L|M)", "v10");
+ mput(m1, "cf1", "cq1", "FOO", "v11");
+ mput(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)", "v12");
+ mput(m1, "cf1", "cq1", "A&B&(L|M|FOO)", "v13");
+
+ bw.addMutation(m1);
+ bw.close();
+ }
+
+ private void deleteData(Connector c, String tableName) throws Exception {
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m1 = new Mutation(new Text("row1"));
+
+ mputDelete(m1, "cf1", "cq1", "");
+ mputDelete(m1, "cf1", "cq1", "A");
+ mputDelete(m1, "cf1", "cq1", "A&B");
+ mputDelete(m1, "cf1", "cq1", "B&(L|M)");
+ mputDelete(m1, "cf1", "cq1", "A&B&(L)");
+ mputDelete(m1, "cf1", "cq1", "A&FOO&(L|M)");
+ mputDelete(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)");
+ mputDelete(m1, "cf1", "cq1", "FOO&A"); // should not delete anything
+
+ bw.addMutation(m1);
+ bw.close();
+
+ Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
+
+ expected.put(nss("A", "L"), nss("v5"));
+ expected.put(nss("A", "M"), nss("v5"));
+ expected.put(nss("B"), nss("v3"));
+ expected.put(nss("Z"), nss());
+ expected.put(nss("A", "B", "L"), nss("v7", "v13"));
+ expected.put(nss("A", "B", "M"), nss("v7", "v13"));
+ expected.put(nss("A", "B", "FOO"), nss("v13"));
+ expected.put(nss("FOO"), nss("v11"));
+ expected.put(nss("A", "FOO"), nss("v9"));
+
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
+ }
+
+ private void insertDefaultData(Connector c, String tableName) throws Exception {
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m1 = new Mutation(new Text("row1"));
+
+ mput(m1, "cf1", "cq1", "BASE", "v1");
+ mput(m1, "cf1", "cq2", "DEFLABEL", "v2");
+ mput(m1, "cf1", "cq3", "", "v3");
+
+ bw.addMutation(m1);
+ bw.close();
+ }
+
+ private static void uniqueCombos(List<Set<String>> all, Set<String> prefix, Set<String> suffix) {
+
+ all.add(prefix);
+
+ TreeSet<String> ss = new TreeSet<String>(suffix);
+
+ for (String s : suffix) {
+ TreeSet<String> ps = new TreeSet<String>(prefix);
+ ps.add(s);
+ ss.remove(s);
+
+ uniqueCombos(all, ps, ss);
+ }
+ }
+
+ private void queryData(Connector c, String tableName) throws Exception {
+ Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
+ expected.put(nss(), nss("v1"));
+ expected.put(nss("A"), nss("v2"));
+ expected.put(nss("A", "L"), nss("v5"));
+ expected.put(nss("A", "M"), nss("v5"));
+ expected.put(nss("B"), nss("v3"));
+ expected.put(nss("B", "L"), nss("v6"));
+ expected.put(nss("B", "M"), nss("v6"));
+ expected.put(nss("Z"), nss());
+ expected.put(nss("A", "B"), nss("v4"));
+ expected.put(nss("A", "B", "L"), nss("v7", "v8", "v13"));
+ expected.put(nss("A", "B", "M"), nss("v7", "v13"));
+ expected.put(nss("A", "B", "FOO"), nss("v13"));
+ expected.put(nss("FOO"), nss("v11"));
+ expected.put(nss("A", "FOO"), nss("v9"));
+ expected.put(nss("A", "FOO", "L"), nss("v10", "v12"));
+ expected.put(nss("A", "FOO", "M"), nss("v10", "v12"));
+ expected.put(nss("B", "FOO", "L"), nss("v12"));
+ expected.put(nss("B", "FOO", "M"), nss("v12"));
+
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "L", "M", "Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss("Z"), expected);
+ queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected);
+ }
+
+ private void queryData(Connector c, String tableName, Set<String> allAuths, Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws Exception {
+
+ c.securityOperations().changeUserAuthorizations("root", new Authorizations(nbas(userAuths)));
+
+ ArrayList<Set<String>> combos = new ArrayList<Set<String>>();
+ uniqueCombos(combos, nss(), allAuths);
+
+ for (Set<String> set1 : combos) {
+ Set<String> e = new TreeSet<String>();
+ for (Set<String> set2 : combos) {
+
+ set2 = new HashSet<String>(set2);
+ set2.retainAll(userAuths);
+
+ if (set1.containsAll(set2) && expected.containsKey(set2)) {
+ e.addAll(expected.get(set2));
+ }
+ }
+
+ set1.retainAll(userAuths);
+ verify(c, tableName, set1, e);
+ }
+
+ }
+
+ private void queryDefaultData(Connector c, String tableName) throws Exception {
+ Scanner scanner;
+
+ // should return no records
+ c.securityOperations().changeUserAuthorizations("root", new Authorizations("BASE", "DEFLABEL"));
+ scanner = getConnector().createScanner(tableName, new Authorizations());
+ verifyDefault(scanner, 0);
+
+ // should return one record
+ scanner = getConnector().createScanner(tableName, new Authorizations("BASE"));
+ verifyDefault(scanner, 1);
+
+ // should return all three records
+ scanner = getConnector().createScanner(tableName, new Authorizations("BASE", "DEFLABEL"));
+ verifyDefault(scanner, 3);
+ }
+
+ private void verifyDefault(Scanner scanner, int expectedCount) throws Exception {
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner)
+ --expectedCount;
+ if (expectedCount != 0)
+ throw new Exception(" expected count !=0 " + expectedCount);
+ }
+
+ private void verify(Connector c, String tableName, Set<String> auths, Set<String> expectedValues) throws Exception {
+ ByteArraySet bas = nbas(auths);
+
+ try {
+ verify(c, tableName, bas, expectedValues.toArray(new String[0]));
+ } catch (Exception e) {
+ throw new Exception("Verification failed auths=" + auths + " exp=" + expectedValues, e);
+ }
+ }
+
+ private ByteArraySet nbas(Set<String> auths) {
+ ByteArraySet bas = new ByteArraySet();
+ for (String auth : auths) {
- bas.add(auth.getBytes(Constants.UTF8));
++ bas.add(auth.getBytes(UTF_8));
+ }
+ return bas;
+ }
+
+ private void verify(Connector c, String tableName, ByteArraySet nss, String... expected) throws Exception {
+ Scanner scanner = c.createScanner(tableName, new Authorizations(nss));
+ verify(scanner.iterator(), expected);
+
+ BatchScanner bs = getConnector().createBatchScanner(tableName, new Authorizations(nss), 3);
+ bs.setRanges(Collections.singleton(new Range()));
+ verify(bs.iterator(), expected);
+ bs.close();
+ }
+
+ private void verify(Iterator<Entry<Key,Value>> iter, String... expected) throws Exception {
+ HashSet<String> valuesSeen = new HashSet<String>();
+
+ while (iter.hasNext()) {
+ Entry<Key,Value> entry = iter.next();
+ if (valuesSeen.contains(entry.getValue().toString())) {
+ throw new Exception("Value seen twice");
+ }
+ valuesSeen.add(entry.getValue().toString());
+ }
+
+ for (String ev : expected) {
+ if (!valuesSeen.remove(ev)) {
+ throw new Exception("Did not see expected value " + ev);
+ }
+ }
+
+ if (valuesSeen.size() != 0) {
+ throw new Exception("Saw more values than expected " + valuesSeen);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
[09/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
index 1226806,0000000..10cf224
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
@@@ -1,617 -1,0 +1,619 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
+import org.apache.accumulo.trace.instrument.TraceExecutorService;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+
+/*
+ * Bulk import makes requests of tablet servers, and those requests can take a
+ * long time. Our communications to the tablet server may fail, so we won't know
+ * the status of the request. The master will repeat failed requests so now
+ * there are multiple requests to the tablet server. The tablet server will not
+ * execute the request multiple times, so long as the marker it wrote in the
+ * metadata table stays there. The master needs to know when all requests have
+ * finished so it can remove the markers. Did it start? Did it finish? We can see
+ * that *a* request completed by seeing the flag written into the metadata
+ * table, but we won't know if some other rogue thread is still waiting to start
+ * a thread and repeat the operation.
+ *
+ * The master can ask the tablet server if it has any requests still running.
+ * Except the tablet server might have some thread about to start a request, but
+ * before it has made any bookkeeping about the request. To prevent problems
+ * like this, an Arbitrator is used. Before starting any new request, the tablet
+ * server checks the Arbitrator to see if the request is still valid.
+ *
+ */
+
+public class BulkImport extends MasterRepo {
+ public static final String FAILURES_TXT = "failures.txt";
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger log = Logger.getLogger(BulkImport.class);
+
+ private String tableId;
+ private String sourceDir;
+ private String errorDir;
+ private boolean setTime;
+
+ public BulkImport(String tableId, String sourceDir, String errorDir, boolean setTime) {
+ this.tableId = tableId;
+ this.sourceDir = sourceDir;
+ this.errorDir = errorDir;
+ this.setTime = setTime;
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+ if (!Utils.getReadLock(tableId, tid).tryLock())
+ return 100;
+
+ Instance instance = HdfsZooInstance.getInstance();
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) == TableState.ONLINE) {
+ long reserve1, reserve2;
+ reserve1 = reserve2 = Utils.reserveHdfsDirectory(sourceDir, tid);
+ if (reserve1 == 0)
+ reserve2 = Utils.reserveHdfsDirectory(errorDir, tid);
+ return reserve2;
+ } else {
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.OFFLINE, null);
+ }
+ }
+
+ @Override
+ //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ log.debug(" tid " + tid + " sourceDir " + sourceDir);
+
+ Utils.getReadLock(tableId, tid).lock();
+
+ // check that the error directory exists and is empty
+ VolumeManager fs = master.getFileSystem();
+
+ Path errorPath = new Path(errorDir);
+ FileStatus errorStatus = null;
+ try {
+ errorStatus = fs.getFileStatus(errorPath);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ if (errorStatus == null)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ + " does not exist");
+ if (!errorStatus.isDir())
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ + " is not a directory");
+ if (fs.listStatus(errorPath).length != 0)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ + " is not empty");
+
+ ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
+
+ // move the files into the directory
+ try {
+ String bulkDir = prepareBulkImport(fs, sourceDir, tableId);
+ log.debug(" tid " + tid + " bulkDir " + bulkDir);
+ return new LoadFiles(tableId, sourceDir, bulkDir, errorDir, setTime);
+ } catch (IOException ex) {
+ log.error("error preparing the bulk import directory", ex);
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_INPUT_DIRECTORY, sourceDir + ": "
+ + ex);
+ }
+ }
+
+ private Path createNewBulkDir(VolumeManager fs, String tableId) throws IOException {
+ Path tempPath = fs.matchingFileSystem(new Path(sourceDir), ServerConstants.getTablesDirs());
+ if (tempPath == null)
+ throw new IllegalStateException(sourceDir + " is not in a known namespace");
+
+ String tableDir = tempPath.toString();
+ if (tableDir == null)
+ throw new IllegalStateException(sourceDir + " is not in a known namespace");
+ Path directory = new Path(tableDir + "/" + tableId);
+ fs.mkdirs(directory);
+
+ // only one should be able to create the lock file
+ // the purpose of the lock file is to avoid a race
+ // condition between the call to fs.exists() and
+ // fs.mkdirs()... if only hadoop had a mkdir() function
+ // that failed when the dir existed
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
+ while (true) {
+ Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
+ if (fs.exists(newBulkDir)) // sanity check
+ throw new IllegalStateException("Dir exist when it should not " + newBulkDir);
+ if (fs.mkdirs(newBulkDir))
+ return newBulkDir;
+ log.warn("Failed to create " + newBulkDir + " for unknown reason");
+
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ private String prepareBulkImport(VolumeManager fs, String dir, String tableId) throws IOException {
+ Path bulkDir = createNewBulkDir(fs, tableId);
+
+ MetadataTableUtil.addBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
+
+ Path dirPath = new Path(dir);
+ FileStatus[] mapFiles = fs.listStatus(dirPath);
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
+ for (FileStatus fileStatus : mapFiles) {
+ String sa[] = fileStatus.getPath().getName().split("\\.");
+ String extension = "";
+ if (sa.length > 1) {
+ extension = sa[sa.length - 1];
+
+ if (!FileOperations.getValidExtensions().contains(extension)) {
+ log.warn(fileStatus.getPath() + " does not have a valid extension, ignoring");
+ continue;
+ }
+ } else {
+ // assume it is a map file
+ extension = Constants.MAPFILE_EXTENSION;
+ }
+
+ if (extension.equals(Constants.MAPFILE_EXTENSION)) {
+ if (!fileStatus.isDir()) {
+ log.warn(fileStatus.getPath() + " is not a map file, ignoring");
+ continue;
+ }
+
+ if (fileStatus.getPath().getName().equals("_logs")) {
+ log.info(fileStatus.getPath() + " is probably a log directory from a map/reduce task, skipping");
+ continue;
+ }
+ try {
+ FileStatus dataStatus = fs.getFileStatus(new Path(fileStatus.getPath(), MapFile.DATA_FILE_NAME));
+ if (dataStatus.isDir()) {
+ log.warn(fileStatus.getPath() + " is not a map file, ignoring");
+ continue;
+ }
+ } catch (FileNotFoundException fnfe) {
+ log.warn(fileStatus.getPath() + " is not a map file, ignoring");
+ continue;
+ }
+ }
+
+ String newName = "I" + namer.getNextName() + "." + extension;
+ Path newPath = new Path(bulkDir, newName);
+ try {
+ fs.rename(fileStatus.getPath(), newPath);
+ log.debug("Moved " + fileStatus.getPath() + " to " + newPath);
+ } catch (IOException E1) {
+ log.error("Could not move: " + fileStatus.getPath().toString() + " " + E1.getMessage());
+ }
+ }
+ return bulkDir.toString();
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ // unreserve source/error directories
+ Utils.unreserveHdfsDirectory(sourceDir, tid);
+ Utils.unreserveHdfsDirectory(errorDir, tid);
+ Utils.getReadLock(tableId, tid).unlock();
+ }
+}
+
+class CleanUpBulkImport extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger log = Logger.getLogger(CleanUpBulkImport.class);
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String error;
+
+ public CleanUpBulkImport(String tableId, String source, String bulk, String error) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.error = error;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ log.debug("removing the bulk processing flag file in " + bulk);
+ Path bulkDir = new Path(bulk);
+ MetadataTableUtil.removeBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
+ MetadataTableUtil.addDeleteEntry(tableId, bulkDir.toString());
+ log.debug("removing the metadata table markers for loaded files");
+ Connector conn = master.getConnector();
+ MetadataTableUtil.removeBulkLoadEntries(conn, tableId, tid);
+ log.debug("releasing HDFS reservations for " + source + " and " + error);
+ Utils.unreserveHdfsDirectory(source, tid);
+ Utils.unreserveHdfsDirectory(error, tid);
+ Utils.getReadLock(tableId, tid).unlock();
+ log.debug("completing bulk import transaction " + tid);
+ ZooArbitrator.cleanup(Constants.BULK_ARBITRATOR_TYPE, tid);
+ return null;
+ }
+}
+
+class CompleteBulkImport extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String error;
+
+ public CompleteBulkImport(String tableId, String source, String bulk, String error) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.error = error;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ ZooArbitrator.stop(Constants.BULK_ARBITRATOR_TYPE, tid);
+ return new CopyFailed(tableId, source, bulk, error);
+ }
+}
+
+class CopyFailed extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String error;
+
+ public CopyFailed(String tableId, String source, String bulk, String error) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.error = error;
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+ Set<TServerInstance> finished = new HashSet<TServerInstance>();
+ Set<TServerInstance> running = master.onlineTabletServers();
+ for (TServerInstance server : running) {
+ try {
+ TServerConnection client = master.getConnection(server);
+ if (client != null && !client.isActive(tid))
+ finished.add(server);
+ } catch (TException ex) {
+ log.info("Ignoring error trying to check on tid " + tid + " from server " + server + ": " + ex);
+ }
+ }
+ if (finished.containsAll(running))
+ return 0;
+ return 500;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ // This needs to execute after the arbiter is stopped
+
+ VolumeManager fs = master.getFileSystem();
+
+ if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
+ return new CleanUpBulkImport(tableId, source, bulk, error);
+
+ HashMap<FileRef,String> failures = new HashMap<FileRef,String>();
+ HashMap<FileRef,String> loadedFailures = new HashMap<FileRef,String>();
+
+ FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
- BufferedReader in = new BufferedReader(new InputStreamReader(failFile, Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(failFile, UTF_8));
+ try {
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ Path path = new Path(line);
+ if (!fs.exists(new Path(error, path.getName())))
+ failures.put(new FileRef(line, path), line);
+ }
+ } finally {
+ failFile.close();
+ }
+
+ /*
+ * I thought I could move files that have no file references in the table. However its possible a clone references a file. Therefore only move files that
+ * have no loaded markers.
+ */
+
+ // determine which failed files were loaded
+ Connector conn = master.getConnector();
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ FileRef loadedFile = new FileRef(fs, entry.getKey());
+ String absPath = failures.remove(loadedFile);
+ if (absPath != null) {
+ loadedFailures.put(loadedFile, absPath);
+ }
+ }
+ }
+
+ // move failed files that were not loaded
+ for (String failure : failures.values()) {
+ Path orig = new Path(failure);
+ Path dest = new Path(error, orig.getName());
+ fs.rename(orig, dest);
+ log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
+ }
+
+ if (loadedFailures.size() > 0) {
+ DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
+ + Constants.ZBULK_FAILED_COPYQ);
+
+ HashSet<String> workIds = new HashSet<String>();
+
+ for (String failure : loadedFailures.values()) {
+ Path orig = new Path(failure);
+ Path dest = new Path(error, orig.getName());
+
+ if (fs.exists(dest))
+ continue;
+
- bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
++ bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8));
+ workIds.add(orig.getName());
+ log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
+ }
+
+ bifCopyQueue.waitUntilDone(workIds);
+ }
+
+ fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
+ return new CleanUpBulkImport(tableId, source, bulk, error);
+ }
+
+}
+
+class LoadFiles extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private static ExecutorService threadPool = null;
+ private static final Logger log = Logger.getLogger(BulkImport.class);
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String errorDir;
+ private boolean setTime;
+
+ public LoadFiles(String tableId, String source, String bulk, String errorDir, boolean setTime) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.errorDir = errorDir;
+ this.setTime = setTime;
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+ if (master.onlineTabletServers().size() == 0)
+ return 500;
+ return 0;
+ }
+
+ private static synchronized ExecutorService getThreadPool(Master master) {
+ if (threadPool == null) {
+ int threadPoolSize = master.getSystemConfiguration().getCount(Property.MASTER_BULK_THREADPOOL_SIZE);
+ ThreadPoolExecutor pool = new SimpleThreadPool(threadPoolSize, "bulk import");
+ pool.allowCoreThreadTimeOut(true);
+ threadPool = new TraceExecutorService(pool);
+ }
+ return threadPool;
+ }
+
+ @Override
+ public Repo<Master> call(final long tid, final Master master) throws Exception {
+ ExecutorService executor = getThreadPool(master);
+ final SiteConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ VolumeManager fs = master.getFileSystem();
+ List<FileStatus> files = new ArrayList<FileStatus>();
+ for (FileStatus entry : fs.listStatus(new Path(bulk))) {
+ files.add(entry);
+ }
+ log.debug("tid " + tid + " importing " + files.size() + " files");
+
+ Path writable = new Path(this.errorDir, ".iswritable");
+ if (!fs.createNewFile(writable)) {
+ // Maybe this is a re-try... clear the flag and try again
+ fs.delete(writable);
+ if (!fs.createNewFile(writable))
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY,
+ "Unable to write to " + this.errorDir);
+ }
+ fs.delete(writable);
+
+ final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<String>());
+ for (FileStatus f : files)
+ filesToLoad.add(f.getPath().toString());
+
+ final int RETRIES = Math.max(1, conf.getCount(Property.MASTER_BULK_RETRIES));
+ for (int attempt = 0; attempt < RETRIES && filesToLoad.size() > 0; attempt++) {
+ List<Future<List<String>>> results = new ArrayList<Future<List<String>>>();
+
+ if (master.onlineTabletServers().size() == 0)
+ log.warn("There are no tablet server to process bulk import, waiting (tid = " + tid + ")");
+
+ while (master.onlineTabletServers().size() == 0) {
+ UtilWaitThread.sleep(500);
+ }
+
+ // Use the threadpool to assign files one-at-a-time to the server
+ final List<String> loaded = Collections.synchronizedList(new ArrayList<String>());
+ for (final String file : filesToLoad) {
+ results.add(executor.submit(new Callable<List<String>>() {
+ @Override
+ public List<String> call() {
+ List<String> failures = new ArrayList<String>();
+ ClientService.Client client = null;
+ String server = null;
+ try {
+ // get a connection to a random tablet server, do not prefer cached connections because
+ // this is running on the master and there are lots of connections to tablet servers
+ // serving the metadata tablets
+ long timeInMillis = master.getConfiguration().getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
+ Pair<String,Client> pair = ServerClient.getConnection(master.getInstance(), false, timeInMillis);
+ client = pair.getSecond();
+ server = pair.getFirst();
+ List<String> attempt = Collections.singletonList(file);
+ log.debug("Asking " + pair.getFirst() + " to bulk import " + file);
+ List<String> fail = client.bulkImportFiles(Tracer.traceInfo(), SystemCredentials.get().toThrift(master.getInstance()), tid, tableId, attempt,
+ errorDir, setTime);
+ if (fail.isEmpty()) {
+ loaded.add(file);
+ } else {
+ failures.addAll(fail);
+ }
+ } catch (Exception ex) {
+ log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
+ } finally {
+ ServerClient.close(client);
+ }
+ return failures;
+ }
+ }));
+ }
+ Set<String> failures = new HashSet<String>();
+ for (Future<List<String>> f : results)
+ failures.addAll(f.get());
+ filesToLoad.removeAll(loaded);
+ if (filesToLoad.size() > 0) {
+ log.debug("tid " + tid + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
+ UtilWaitThread.sleep(100);
+ }
+ }
+
+ FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
- BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, Constants.UTF8));
++ BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8));
+ try {
+ for (String f : filesToLoad) {
+ out.write(f);
+ out.write("\n");
+ }
+ } finally {
+ out.close();
+ }
+
+ // return the next step, which will perform cleanup
+ return new CompleteBulkImport(tableId, source, bulk, errorDir);
+ }
+
+ static String sampleList(Collection<?> potentiallyLongList, int max) {
+ StringBuffer result = new StringBuffer();
+ result.append("[");
+ int i = 0;
+ for (Object obj : potentiallyLongList) {
+ result.append(obj);
+ if (i >= max) {
+ result.append("...");
+ break;
+ } else {
+ result.append(", ");
+ }
+ i++;
+ }
+ if (i < max)
+ result.delete(result.length() - 2, result.length());
+ result.append("]");
+ return result.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
index ebceaa0,0000000..fd2441c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
@@@ -1,105 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+class FinishCancelCompaction extends MasterRepo {
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+
+ public FinishCancelCompaction(String tableId) {
+ this.tableId = tableId;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ Utils.getReadLock(tableId, tid).unlock();
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+
+ }
+}
+
+/**
+ *
+ */
+public class CancelCompactions extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+ private String namespaceId;
+
+ public CancelCompactions(String tableId) {
+ this.tableId = tableId;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT_CANCEL)
+ + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT_CANCEL);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ String zCompactID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+ String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ byte[] currentValue = zoo.getData(zCompactID, null);
+
- String cvs = new String(currentValue, Constants.UTF8);
++ String cvs = new String(currentValue, UTF_8);
+ String[] tokens = cvs.split(",");
+ final long flushID = Long.parseLong(tokens[0]);
+
+ zoo.mutate(zCancelID, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- long cid = Long.parseLong(new String(currentValue, Constants.UTF8));
++ long cid = Long.parseLong(new String(currentValue, UTF_8));
+
+ if (cid < flushID)
- return Long.toString(flushID).getBytes(Constants.UTF8);
++ return Long.toString(flushID).getBytes(UTF_8);
+ else
- return Long.toString(cid).getBytes(Constants.UTF8);
++ return Long.toString(cid).getBytes(UTF_8);
+ }
+ });
+
+ return new FinishCancelCompaction(tableId);
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ Utils.unreserveNamespace(namespaceId, tid, false);
+ Utils.unreserveTable(tableId, tid, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index a643ccb,0000000..fbaa70d
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@@ -1,400 -1,0 +1,402 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+class CompactionDriver extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private long compactId;
+ private String tableId;
+ private byte[] startRow;
+ private byte[] endRow;
+ private String namespaceId;
+
+ public CompactionDriver(long compactId, String tableId, byte[] startRow, byte[] endRow) {
+
+ this.compactId = compactId;
+ this.tableId = tableId;
+ this.startRow = startRow;
+ this.endRow = endRow;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+
+ String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ if (Long.parseLong(new String(zoo.getData(zCancelID, null))) >= compactId) {
+ // compaction was canceled
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, "Compaction canceled");
+ }
+
+ MapCounter<TServerInstance> serversToFlush = new MapCounter<TServerInstance>();
+ Connector conn = master.getConnector();
+
+ Scanner scanner;
+
+ if (tableId.equals(MetadataTable.ID)) {
+ scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ } else {
+ scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+ Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange();
+ scanner.setRange(range);
+ }
+
+ TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner);
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+
+ long t1 = System.currentTimeMillis();
+ RowIterator ri = new RowIterator(scanner);
+
+ int tabletsToWaitFor = 0;
+ int tabletCount = 0;
+
+ while (ri.hasNext()) {
+ Iterator<Entry<Key,Value>> row = ri.next();
+ long tabletCompactID = -1;
+
+ TServerInstance server = null;
+
+ Entry<Key,Value> entry = null;
+ while (row.hasNext()) {
+ entry = row.next();
+ Key key = entry.getKey();
+
+ if (TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ tabletCompactID = Long.parseLong(entry.getValue().toString());
+
+ if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily()))
+ server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
+ }
+
+ if (tabletCompactID < compactId) {
+ tabletsToWaitFor++;
+ if (server != null)
+ serversToFlush.increment(server, 1);
+ }
+
+ tabletCount++;
+
+ Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
+ if (tabletEndRow == null || (endRow != null && tabletEndRow.compareTo(new Text(endRow)) >= 0))
+ break;
+ }
+
+ long scanTime = System.currentTimeMillis() - t1;
+
+ Instance instance = master.getInstance();
+ Tables.clearCache(instance);
+ if (tabletCount == 0 && !Tables.exists(instance, tableId))
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
+
+ if (serversToFlush.size() == 0 && Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
+
+ if (tabletsToWaitFor == 0)
+ return 0;
+
+ for (TServerInstance tsi : serversToFlush.keySet()) {
+ try {
+ final TServerConnection server = master.getConnection(tsi);
+ if (server != null)
+ server.compact(master.getMasterLock(), tableId, startRow, endRow);
+ } catch (TException ex) {
+ Logger.getLogger(CompactionDriver.class).error(ex.toString());
+ }
+ }
+
+ long sleepTime = 500;
+
+ if (serversToFlush.size() > 0)
+ sleepTime = Collections.max(serversToFlush.values()) * sleepTime; // make wait time depend on the server with the most to
+ // compact
+
+ sleepTime = Math.max(2 * scanTime, sleepTime);
+
+ sleepTime = Math.min(sleepTime, 30000);
+
+ return sleepTime;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ CompactRange.removeIterators(tid, tableId);
+ Utils.getReadLock(tableId, tid).unlock();
+ Utils.getReadLock(namespaceId, tid).unlock();
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+
+ }
+
+}
+
+public class CompactRange extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+ private byte[] startRow;
+ private byte[] endRow;
+ private byte[] iterators;
+ private String namespaceId;
+
+ public static class CompactionIterators implements Writable {
+ byte[] startRow;
+ byte[] endRow;
+ List<IteratorSetting> iterators;
+
+ public CompactionIterators(byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) {
+ this.startRow = startRow;
+ this.endRow = endRow;
+ this.iterators = iterators;
+ }
+
+ public CompactionIterators() {
+ startRow = null;
+ endRow = null;
+ iterators = Collections.emptyList();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(startRow != null);
+ if (startRow != null) {
+ out.writeInt(startRow.length);
+ out.write(startRow);
+ }
+
+ out.writeBoolean(endRow != null);
+ if (endRow != null) {
+ out.writeInt(endRow.length);
+ out.write(endRow);
+ }
+
+ out.writeInt(iterators.size());
+ for (IteratorSetting is : iterators) {
+ is.write(out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ if (in.readBoolean()) {
+ startRow = new byte[in.readInt()];
+ in.readFully(startRow);
+ } else {
+ startRow = null;
+ }
+
+ if (in.readBoolean()) {
+ endRow = new byte[in.readInt()];
+ in.readFully(endRow);
+ } else {
+ endRow = null;
+ }
+
+ int num = in.readInt();
+ iterators = new ArrayList<IteratorSetting>(num);
+
+ for (int i = 0; i < num; i++) {
+ iterators.add(new IteratorSetting(in));
+ }
+ }
+
+ public Text getEndRow() {
+ if (endRow == null)
+ return null;
+ return new Text(endRow);
+ }
+
+ public Text getStartRow() {
+ if (startRow == null)
+ return null;
+ return new Text(startRow);
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+ }
+
+ public CompactRange(String tableId, byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) throws ThriftTableOperationException {
+ this.tableId = tableId;
+ this.startRow = startRow.length == 0 ? null : startRow;
+ this.endRow = endRow.length == 0 ? null : endRow;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+
+ if (iterators.size() > 0) {
+ this.iterators = WritableUtils.toByteArray(new CompactionIterators(this.startRow, this.endRow, iterators));
+ } else {
+ iterators = null;
+ }
+
+ if (this.startRow != null && this.endRow != null && new Text(startRow).compareTo(new Text(endRow)) >= 0)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.BAD_RANGE,
+ "start row must be less than end row");
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT)
+ + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT);
+ }
+
+ @Override
+ public Repo<Master> call(final long tid, Master environment) throws Exception {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ byte[] cid;
+ try {
+ cid = zoo.mutate(zTablePath, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- String cvs = new String(currentValue, Constants.UTF8);
++ String cvs = new String(currentValue, UTF_8);
+ String[] tokens = cvs.split(",");
+ long flushID = Long.parseLong(tokens[0]);
+ flushID++;
+
+ String txidString = String.format("%016x", tid);
+
+ for (int i = 1; i < tokens.length; i++) {
+ if (tokens[i].startsWith(txidString))
+ continue; // skip self
+
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER,
+ "Another compaction with iterators is running");
+ }
+
+ StringBuilder encodedIterators = new StringBuilder();
+
+ if (iterators != null) {
+ Hex hex = new Hex();
+ encodedIterators.append(",");
+ encodedIterators.append(txidString);
+ encodedIterators.append("=");
- encodedIterators.append(new String(hex.encode(iterators), Constants.UTF8));
++ encodedIterators.append(new String(hex.encode(iterators), UTF_8));
+ }
+
- return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++ return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
+ }
+ });
+
- return new CompactionDriver(Long.parseLong(new String(cid, Constants.UTF8).split(",")[0]), tableId, startRow, endRow);
++ return new CompactionDriver(Long.parseLong(new String(cid, UTF_8).split(",")[0]), tableId, startRow, endRow);
+ } catch (NoNodeException nne) {
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
+ }
+
+ }
+
+ static void removeIterators(final long txid, String tableId) throws Exception {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ zoo.mutate(zTablePath, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- String cvs = new String(currentValue, Constants.UTF8);
++ String cvs = new String(currentValue, UTF_8);
+ String[] tokens = cvs.split(",");
+ long flushID = Long.parseLong(tokens[0]);
+
+ String txidString = String.format("%016x", txid);
+
+ StringBuilder encodedIterators = new StringBuilder();
+ for (int i = 1; i < tokens.length; i++) {
+ if (tokens[i].startsWith(txidString))
+ continue;
+ encodedIterators.append(",");
+ encodedIterators.append(tokens[i]);
+ }
+
- return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++ return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
+ }
+ });
+
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ try {
+ removeIterators(tid, tableId);
+ } finally {
+ Utils.unreserveNamespace(namespaceId, tid, false);
+ Utils.unreserveTable(tableId, tid, false);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index eeb9b16,0000000..037455c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@@ -1,316 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+class ExportInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public String tableName;
+ public String tableID;
+ public String exportDir;
+ public String namespaceID;
+}
+
+class WriteExportFiles extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private final ExportInfo tableInfo;
+
+ WriteExportFiles(ExportInfo tableInfo) {
+ this.tableInfo = tableInfo;
+ }
+
+ private void checkOffline(Connector conn) throws Exception {
+ if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
+ Tables.clearCache(conn.getInstance());
+ if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
+ throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
+ "Table is not offline");
+ }
+ }
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+
+ long reserved = Utils.reserveNamespace(tableInfo.namespaceID, tid, false, true, TableOperation.EXPORT)
+ + Utils.reserveTable(tableInfo.tableID, tid, false, true, TableOperation.EXPORT);
+ if (reserved > 0)
+ return reserved;
+
+ Connector conn = master.getConnector();
+
+ checkOffline(conn);
+
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.setRange(new KeyExtent(new Text(tableInfo.tableID), null, null).toMetadataRange());
+
+ // scan for locations
+ metaScanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ metaScanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
+
+ if (metaScanner.iterator().hasNext()) {
+ return 500;
+ }
+
+ // use the same range to check for walogs that we used to check for hosted (or future hosted) tablets
+ // this is done as a separate scan after we check for locations, because walogs are okay only if there is no location
+ metaScanner.clearColumns();
+ metaScanner.fetchColumnFamily(LogColumnFamily.NAME);
+
+ if (metaScanner.iterator().hasNext()) {
+ throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
+ "Write ahead logs found for table");
+ }
+
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ Connector conn = master.getConnector();
+
+ try {
+ exportTable(master.getFileSystem(), conn, tableInfo.tableName, tableInfo.tableID, tableInfo.exportDir);
+ } catch (IOException ioe) {
+ throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
+ "Failed to create export files " + ioe.getMessage());
+ }
+ Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
+ Utils.unreserveTable(tableInfo.tableID, tid, false);
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
+ Utils.unreserveTable(tableInfo.tableID, tid, false);
+ }
+
+ public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
+
+ fs.mkdirs(new Path(exportDir));
+ Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
+
+ FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
+ ZipOutputStream zipOut = new ZipOutputStream(fileOut);
+ BufferedOutputStream bufOut = new BufferedOutputStream(zipOut);
+ DataOutputStream dataOut = new DataOutputStream(bufOut);
+
+ try {
+
+ zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_INFO_FILE));
- OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++ OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
+ osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.VERSION + "\n");
+ osw.append("srcInstanceName:" + conn.getInstance().getInstanceName() + "\n");
+ osw.append("srcInstanceID:" + conn.getInstance().getInstanceID() + "\n");
+ osw.append("srcZookeepers:" + conn.getInstance().getZooKeepers() + "\n");
+ osw.append("srcTableName:" + tableName + "\n");
+ osw.append("srcTableID:" + tableID + "\n");
+ osw.append(ExportTable.DATA_VERSION_PROP + ":" + ServerConstants.DATA_VERSION + "\n");
+ osw.append("srcCodeVersion:" + Constants.VERSION + "\n");
+
+ osw.flush();
+ dataOut.flush();
+
+ exportConfig(conn, tableID, zipOut, dataOut);
+ dataOut.flush();
+
+ Map<String,String> uniqueFiles = exportMetadata(fs, conn, tableID, zipOut, dataOut);
+
+ dataOut.close();
+ dataOut = null;
+
+ createDistcpFile(fs, exportDir, exportMetaFilePath, uniqueFiles);
+
+ } finally {
+ if (dataOut != null)
+ dataOut.close();
+ }
+ }
+
+ private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException {
- BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), Constants.UTF8));
++ BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), UTF_8));
+
+ try {
+ for (String file : uniqueFiles.values()) {
+ distcpOut.append(file);
+ distcpOut.newLine();
+ }
+
+ distcpOut.append(exportMetaFilePath.toString());
+ distcpOut.newLine();
+
+ distcpOut.close();
+ distcpOut = null;
+
+ } finally {
+ if (distcpOut != null)
+ distcpOut.close();
+ }
+ }
+
+ private static Map<String,String> exportMetadata(VolumeManager fs, Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut)
+ throws IOException, TableNotFoundException {
+ zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_METADATA_FILE));
+
+ Map<String,String> uniqueFiles = new HashMap<String,String>();
+
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(metaScanner);
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(metaScanner);
+ metaScanner.setRange(new KeyExtent(new Text(tableID), null, null).toMetadataRange());
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ entry.getKey().write(dataOut);
+ entry.getValue().write(dataOut);
+
+ if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String path = fs.getFullPath(entry.getKey()).toString();
+ String tokens[] = path.split("/");
+ if (tokens.length < 1) {
+ throw new RuntimeException("Illegal path " + path);
+ }
+
+ String filename = tokens[tokens.length - 1];
+
+ String existingPath = uniqueFiles.get(filename);
+ if (existingPath == null) {
+ uniqueFiles.put(filename, path);
+ } else if (!existingPath.equals(path)) {
+ // make sure file names are unique, should only apply for tables with file names generated by Accumulo 1.3 and earlier
+ throw new IOException("Cannot export table with nonunique file names " + filename + ". Major compact table.");
+ }
+
+ }
+ }
+ return uniqueFiles;
+ }
+
+ private static void exportConfig(Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws AccumuloException,
+ AccumuloSecurityException, TableNotFoundException, IOException {
+
+ DefaultConfiguration defaultConfig = AccumuloConfiguration.getDefaultConfiguration();
+ Map<String,String> siteConfig = conn.instanceOperations().getSiteConfiguration();
+ Map<String,String> systemConfig = conn.instanceOperations().getSystemConfiguration();
+
+ TableConfiguration tableConfig = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableID);
+
- OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++ OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
+
+ // only put props that are different than defaults and higher level configurations
+ zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));
+ for (Entry<String,String> prop : tableConfig) {
+ if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) {
+ Property key = Property.getPropertyByKey(prop.getKey());
+
+ if (key == null || !defaultConfig.get(key).equals(prop.getValue())) {
+ if (!prop.getValue().equals(siteConfig.get(prop.getKey())) && !prop.getValue().equals(systemConfig.get(prop.getKey()))) {
+ osw.append(prop.getKey() + "=" + prop.getValue() + "\n");
+ }
+ }
+ }
+ }
+
+ osw.flush();
+ }
+}
+
+public class ExportTable extends MasterRepo {
+ private static final long serialVersionUID = 1L;
+
+ private final ExportInfo tableInfo;
+
+ public ExportTable(String tableName, String tableId, String exportDir) {
+ tableInfo = new ExportInfo();
+ tableInfo.tableName = tableName;
+ tableInfo.exportDir = exportDir;
+ tableInfo.tableID = tableId;
+ Instance inst = HdfsZooInstance.getInstance();
+ tableInfo.namespaceID = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ return new WriteExportFiles(tableInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ }
+
+ public static final int VERSION = 1;
+
+ public static final String DATA_VERSION_PROP = "srcDataVersion";
+ public static final String EXPORT_VERSION_PROP = "exportVersion";
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 8de6bce,0000000..05ec263
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@@ -1,653 -1,0 +1,655 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.TableOperationsImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+class ImportedTableInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public String exportDir;
+ public String user;
+ public String tableName;
+ public String tableId;
+ public String importDir;
+ public String namespaceId;
+}
+
+class FinishImportTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public FinishImportTable(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+
+ env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir, "mappings.txt"));
+
+ TableManager.getInstance().transitionTableState(tableInfo.tableId, TableState.ONLINE);
+
+ Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
+ Utils.unreserveTable(tableInfo.tableId, tid, true);
+
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+
+ env.getEventCoordinator().event("Imported table %s ", tableInfo.tableName);
+
+ Logger.getLogger(FinishImportTable.class).debug("Imported table " + tableInfo.tableId + " " + tableInfo.tableName);
+
+ return null;
+ }
+
+ @Override
+ public String getReturn() {
+ return tableInfo.tableId;
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {}
+
+}
+
+class MoveExportedFiles extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ MoveExportedFiles(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ try {
+ VolumeManager fs = master.getFileSystem();
+
+ Map<String,String> fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo);
+
+ for (String oldFileName : fileNameMappings.keySet()) {
+ if (!fs.exists(new Path(tableInfo.exportDir, oldFileName))) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "File referenced by exported table does not exists " + oldFileName);
+ }
+ }
+
+ FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
+
+ for (FileStatus fileStatus : files) {
+ String newName = fileNameMappings.get(fileStatus.getPath().getName());
+
+ if (newName != null)
+ fs.rename(fileStatus.getPath(), new Path(tableInfo.importDir, newName));
+ }
+
+ return new FinishImportTable(tableInfo);
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error renaming files " + ioe.getMessage());
+ }
+ }
+}
+
+class PopulateMetadataTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ PopulateMetadataTable(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ static Map<String,String> readMappingFile(VolumeManager fs, ImportedTableInfo tableInfo) throws Exception {
- BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), UTF_8));
+
+ try {
+ Map<String,String> map = new HashMap<String,String>();
+
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split(":", 2);
+ map.put(sa[0], sa[1]);
+ }
+
+ return map;
+ } finally {
+ in.close();
+ }
+
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+
+ Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
+
+ BatchWriter mbw = null;
+ ZipInputStream zis = null;
+
+ try {
+ VolumeManager fs = master.getFileSystem();
+
+ mbw = master.getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ zis = new ZipInputStream(fs.open(path));
+
+ Map<String,String> fileNameMappings = readMappingFile(fs, tableInfo);
+
+ log.info("importDir is " + tableInfo.importDir);
+
+ // This is a directory already prefixed with proper volume information e.g. hdfs://localhost:8020/path/to/accumulo/tables/...
+ final String bulkDir = tableInfo.importDir;
+
+ final String[] tableDirs = ServerConstants.getTablesDirs();
+
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) {
+ DataInputStream in = new DataInputStream(new BufferedInputStream(zis));
+
+ Key key = new Key();
+ Value val = new Value();
+
+ Mutation m = null;
+ Text currentRow = null;
+ int dirCount = 0;
+
+ while (true) {
+ key.readFields(in);
+ val.readFields(in);
+
+ Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow();
+ Text metadataRow = new KeyExtent(new Text(tableInfo.tableId), endRow, null).getMetadataEntry();
+
+ Text cq;
+
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String oldName = new Path(key.getColumnQualifier().toString()).getName();
+ String newName = fileNameMappings.get(oldName);
+
+ if (newName == null) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "File " + oldName + " does not exist in import dir");
+ }
+
+ cq = new Text(bulkDir + "/" + newName);
+ } else {
+ cq = key.getColumnQualifier();
+ }
+
+ if (m == null) {
+ // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
- String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++ String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
+
+ // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
+ String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+
+ m = new Mutation(metadataRow);
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
+ currentRow = metadataRow;
+ }
+
+ if (!currentRow.equals(metadataRow)) {
+ mbw.addMutation(m);
+
+ // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
- String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++ String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
+
+ // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
+ String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+
+ m = new Mutation(metadataRow);
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
+ }
+
+ m.put(key.getColumnFamily(), cq, val);
+
+ if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
+ mbw.addMutation(m);
+ break; // its the last column in the last row
+ }
+ }
+
+ break;
+ }
+ }
+
+ return new MoveExportedFiles(tableInfo);
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error reading " + path + " " + ioe.getMessage());
+ } finally {
+ if (zis != null) {
+ try {
+ zis.close();
+ } catch (IOException ioe) {
+ log.warn("Failed to close zip file ", ioe);
+ }
+ }
+
+ if (mbw != null) {
+ mbw.close();
+ }
+ }
+ }
+
+ /**
+ * Given options for tables (across multiple volumes), construct an absolute path using the unique name within the chosen volume
+ *
+ * @return An absolute, unique path for the imported table
+ */
+ protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) {
+ // We can try to spread out the tablet dirs across all volumes
+ String tableDir = master.getFileSystem().choose(tableDirs);
+
+ // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
+ return tableDir + "/" + tableInfo.tableId + "/" + tabletDir;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ MetadataTableUtil.deleteTable(tableInfo.tableId, false, SystemCredentials.get(), environment.getMasterLock());
+ }
+}
+
+class MapImportFileNames extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ MapImportFileNames(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+
+ Path path = new Path(tableInfo.importDir, "mappings.txt");
+
+ BufferedWriter mappingsWriter = null;
+
+ try {
+ VolumeManager fs = environment.getFileSystem();
+
+ fs.mkdirs(new Path(tableInfo.importDir));
+
+ FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
- mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), Constants.UTF8));
++ mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), UTF_8));
+
+ for (FileStatus fileStatus : files) {
+ String fileName = fileStatus.getPath().getName();
+ log.info("filename " + fileStatus.getPath().toString());
+ String sa[] = fileName.split("\\.");
+ String extension = "";
+ if (sa.length > 1) {
+ extension = sa[sa.length - 1];
+
+ if (!FileOperations.getValidExtensions().contains(extension)) {
+ continue;
+ }
+ } else {
+ // assume it is a map file
+ extension = Constants.MAPFILE_EXTENSION;
+ }
+
+ String newName = "I" + namer.getNextName() + "." + extension;
+
+ mappingsWriter.append(fileName);
+ mappingsWriter.append(':');
+ mappingsWriter.append(newName);
+ mappingsWriter.newLine();
+ }
+
+ mappingsWriter.close();
+ mappingsWriter = null;
+
+ return new PopulateMetadataTable(tableInfo);
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error writing mapping file " + path + " " + ioe.getMessage());
+ } finally {
+ if (mappingsWriter != null)
+ try {
+ mappingsWriter.close();
+ } catch (IOException ioe) {
+ log.warn("Failed to close " + path, ioe);
+ }
+ }
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir));
+ }
+}
+
+class CreateImportDir extends MasterRepo {
+ private static final Logger log = Logger.getLogger(CreateImportDir.class);
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ CreateImportDir(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
+ Path exportDir = new Path(tableInfo.exportDir);
+ String[] tableDirs = ServerConstants.getTablesDirs();
+
+ log.info("Looking for matching filesystem for " + exportDir + " from options " + Arrays.toString(tableDirs));
+ Path base = master.getFileSystem().matchingFileSystem(exportDir, tableDirs);
+ log.info("Chose base table directory of " + base);
+ Path directory = new Path(base, tableInfo.tableId);
+
+ Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
+
+ tableInfo.importDir = newBulkDir.toString();
+
+ log.info("Using import dir: " + tableInfo.importDir);
+
+ return new MapImportFileNames(tableInfo);
+ }
+}
+
+class ImportPopulateZookeeper extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ ImportPopulateZookeeper(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT);
+ }
+
+ private Map<String,String> getExportedProps(VolumeManager fs) throws Exception {
+
+ Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
+
+ try {
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ return TableOperationsImpl.getExportedProps(ns, path);
+ } catch (IOException ioe) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error reading table props from " + path + " " + ioe.getMessage());
+ }
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ // reserve the table name in zookeeper or fail
+
+ Utils.tableNameLock.lock();
+ try {
+ // write tableName & tableId to zookeeper
+ Instance instance = HdfsZooInstance.getInstance();
+
+ Utils.checkTableDoesNotExist(instance, tableInfo.tableName, tableInfo.tableId, TableOperation.CREATE);
+
+ String namespace = Tables.qualify(tableInfo.tableName).getFirst();
+ String namespaceId = Namespaces.getNamespaceId(instance, namespace);
+ TableManager.getInstance().addTable(tableInfo.tableId, namespaceId, tableInfo.tableName, NodeExistsPolicy.OVERWRITE);
+
+ Tables.clearCache(instance);
+ } finally {
+ Utils.tableNameLock.unlock();
+ }
+
+ for (Entry<String,String> entry : getExportedProps(env.getFileSystem()).entrySet())
+ if (!TablePropUtil.setTableProperty(tableInfo.tableId, entry.getKey(), entry.getValue())) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Invalid table property " + entry.getKey());
+ }
+
+ return new CreateImportDir(tableInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Instance instance = HdfsZooInstance.getInstance();
+ TableManager.getInstance().removeTable(tableInfo.tableId);
+ Utils.unreserveTable(tableInfo.tableId, tid, true);
+ Tables.clearCache(instance);
+ }
+}
+
+class ImportSetupPermissions extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public ImportSetupPermissions(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ // give all table permissions to the creator
+ SecurityOperation security = AuditedSecurityOperation.getInstance();
+ for (TablePermission permission : TablePermission.values()) {
+ try {
+ security.grantTablePermission(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.user, tableInfo.tableId, permission, tableInfo.namespaceId);
+ } catch (ThriftSecurityException e) {
+ Logger.getLogger(ImportSetupPermissions.class).error(e.getMessage(), e);
+ throw e;
+ }
+ }
+
+ // setup permissions in zookeeper before table info in zookeeper
+ // this way concurrent users will not get a spurious permission denied
+ // error
+ return new ImportPopulateZookeeper(tableInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ AuditedSecurityOperation.getInstance().deleteTable(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.tableId, tableInfo.namespaceId);
+ }
+}
+
+public class ImportTable extends MasterRepo {
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public ImportTable(String user, String tableName, String exportDir, String namespaceId) {
+ tableInfo = new ImportedTableInfo();
+ tableInfo.tableName = tableName;
+ tableInfo.user = user;
+ tableInfo.exportDir = exportDir;
+ tableInfo.namespaceId = namespaceId;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid)
+ + Utils.reserveNamespace(tableInfo.namespaceId, tid, false, true, TableOperation.IMPORT);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ checkVersions(env);
+
+ // first step is to reserve a table id.. if the machine fails during this step
+ // it is ok to retry... the only side effect is that a table id may not be used
+ // or skipped
+
+ // assuming only the master process is creating tables
+
+ Utils.idLock.lock();
+ try {
+ Instance instance = HdfsZooInstance.getInstance();
+ tableInfo.tableId = Utils.getNextTableId(tableInfo.tableName, instance);
+ return new ImportSetupPermissions(tableInfo);
+ } finally {
+ Utils.idLock.unlock();
+ }
+ }
+
+ public void checkVersions(Master env) throws ThriftTableOperationException {
+ Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
+
+ ZipInputStream zis = null;
+
+ try {
+ zis = new ZipInputStream(env.getFileSystem().open(path));
+
+ Integer exportVersion = null;
+ Integer dataVersion = null;
+
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_INFO_FILE)) {
- BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split(":", 2);
+ if (sa[0].equals(ExportTable.EXPORT_VERSION_PROP)) {
+ exportVersion = Integer.parseInt(sa[1]);
+ } else if (sa[0].equals(ExportTable.DATA_VERSION_PROP)) {
+ dataVersion = Integer.parseInt(sa[1]);
+ }
+ }
+
+ break;
+ }
+ }
+
+ zis.close();
+ zis = null;
+
+ if (exportVersion == null || exportVersion > ExportTable.VERSION)
+ throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Incompatible export version " + exportVersion);
+
+ if (dataVersion == null || dataVersion > ServerConstants.DATA_VERSION)
+ throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Incompatible data version " + exportVersion);
+
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Failed to read export metadata " + ioe.getMessage());
+ } finally {
+ if (zis != null)
+ try {
+ zis.close();
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ }
+ }
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
+ }
+}
[11/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index 292129d,0000000..7ae4d5a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@@ -1,150 -1,0 +1,152 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.watcher;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Appender;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.FileWatchdog;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * Watcher that updates the monitor's log4j port from ZooKeeper in a system property
+ */
+public class MonitorLog4jWatcher extends FileWatchdog implements Watcher {
+ private static final Logger log = Logger.getLogger(MonitorLog4jWatcher.class);
+
+ private static final String HOST_PROPERTY_NAME = "org.apache.accumulo.core.host.log";
+ private static final String PORT_PROPERTY_NAME = "org.apache.accumulo.core.host.log.port";
+
+ private final Object lock;
+ private final Log4jConfiguration logConfig;
+ private boolean loggingDisabled = false;
+ protected String path;
+
+ public MonitorLog4jWatcher(String instance, String filename) {
+ super(filename);
+ this.path = ZooUtil.getRoot(instance) + Constants.ZMONITOR_LOG4J_ADDR;
+ this.lock = new Object();
+ this.logConfig = new Log4jConfiguration(filename);
+ doOnChange();
+ }
+
+ boolean isUsingProperties() {
+ return logConfig.isUsingProperties();
+ }
+
+ String getPath() {
+ return path;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // Initially set the logger if the Monitor's log4j advertisement node exists
+ if (ZooReaderWriter.getInstance().exists(path, this))
+ updateMonitorLog4jLocation();
+ log.info("Set watch for Monitor Log4j watcher");
+ } catch (Exception e) {
+ log.error("Unable to set watch for Monitor Log4j watcher on " + path);
+ }
+
+ super.run();
+ }
+
+ @Override
+ public void doOnChange() {
+ // this method gets called in the parent class' constructor
+ // I'm not sure of a better way to get around this. The final modifier helps though.
+ if (null == lock) {
+ return;
+ }
+
+ synchronized (lock) {
+ // We might triggered by file-reloading or from ZK update.
+ // Either way will result in log-forwarding being restarted
+ loggingDisabled = false;
+ log.info("Enabled log-forwarding");
+ logConfig.resetLogger();
+ }
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ // We got an update, process the data in the node
+ updateMonitorLog4jLocation();
+
+ if (event.getPath() != null) {
+ try {
+ ZooReaderWriter.getInstance().exists(event.getPath(), this);
+ } catch (Exception ex) {
+ log.error("Unable to reset watch for Monitor Log4j watcher", ex);
+ }
+ }
+ }
+
+ /**
+ * Read the host and port information for the Monitor's log4j socket and update the system properties so that, on logger refresh, it sees the new information.
+ */
+ protected void updateMonitorLog4jLocation() {
+ try {
- String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), Constants.UTF8);
++ String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), UTF_8);
+ HostAndPort hostAndPort = HostAndPort.fromString(hostPortString);
+
+ System.setProperty(HOST_PROPERTY_NAME, hostAndPort.getHostText());
+ System.setProperty(PORT_PROPERTY_NAME, Integer.toString(hostAndPort.getPort()));
+
+ log.info("Changing monitor log4j address to " + hostAndPort.toString());
+
+ doOnChange();
+ } catch (NoNodeException e) {
+ // Not sure on the synchronization guarantees for Loggers and Appenders
+ // on configuration reload
+ synchronized (lock) {
+ // Don't need to try to re-disable'ing it.
+ if (loggingDisabled) {
+ return;
+ }
+
+ Logger logger = LogManager.getLogger("org.apache.accumulo");
+ if (null != logger) {
+ // TODO ACCUMULO-2343 Create a specific appender for log-forwarding to the monitor
+ // that can replace the AsyncAppender+SocketAppender.
+ Appender appender = logger.getAppender("ASYNC");
+ if (null != appender) {
+ log.info("Closing log-forwarding appender");
+ appender.close();
+ log.info("Removing log-forwarding appender");
+ logger.removeAppender(appender);
+ loggingDisabled = true;
+ }
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ log.error("Could not parse host and port information", e);
+ } catch (Exception e) {
+ log.error("Error reading zookeeper data for Monitor Log4j watcher", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
index b02e5d4,0000000..1bbae03
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
@@@ -1,54 -1,0 +1,55 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
+import org.apache.zookeeper.KeeperException;
+
+public class ZooQueueLock extends org.apache.accumulo.fate.zookeeper.ZooQueueLock {
+
+ public ZooQueueLock(String path, boolean ephemeral) throws KeeperException, InterruptedException {
+ super(ZooReaderWriter.getInstance(), path, ephemeral);
+ }
+
+ public static void main(String args[]) throws InterruptedException, KeeperException {
+ ZooQueueLock lock = new ZooQueueLock("/lock", true);
- DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(Constants.UTF8));
- DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(Constants.UTF8));
++ DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(UTF_8));
++ DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(UTF_8));
+ final Lock readLock = rlocker.readLock();
+ readLock.lock();
+ final Lock readLock2 = rlocker.readLock();
+ readLock2.lock();
+ final Lock writeLock = wlocker.writeLock();
+ if (writeLock.tryLock(100, TimeUnit.MILLISECONDS))
+ throw new RuntimeException("Write lock achieved during read lock!");
+ readLock.unlock();
+ readLock2.unlock();
+ writeLock.lock();
+ if (readLock.tryLock(100, TimeUnit.MILLISECONDS))
+ throw new RuntimeException("Read lock achieved during write lock!");
- final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(Constants.UTF8));
++ final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(UTF_8));
+ writeLock2.unlock();
+ readLock.lock();
+ System.out.println("success");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
index 435591d,0000000..c5374a2
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
@@@ -1,42 -1,0 +1,43 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+
+public class ZooReaderWriter extends org.apache.accumulo.fate.zookeeper.ZooReaderWriter {
+ private static final String SCHEME = "digest";
+ private static final String USER = "accumulo";
+ private static ZooReaderWriter instance = null;
+
+ public ZooReaderWriter(String string, int timeInMillis, String secret) {
- super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++ super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
+ }
+
+ public static synchronized ZooReaderWriter getInstance() {
+ if (instance == null) {
+ AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ instance = new ZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
+ conf.get(Property.INSTANCE_SECRET));
+ }
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index 4a7111d,0000000..e15e63e
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@@ -1,64 -1,0 +1,65 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+
+/**
+ * A factory for {@link ZooReaderWriter} objects.
+ */
+public class ZooReaderWriterFactory {
+ private static final String SCHEME = "digest";
+ private static final String USER = "accumulo";
+ private static IZooReaderWriter instance = null;
+
+ /**
+ * Gets a new reader/writer.
+ *
+ * @param string
+ * ZooKeeper connection string
+ * @param timeInMillis
+ * session timeout in milliseconds
+ * @param secret
+ * instance secret
+ * @return reader/writer
+ */
+ public IZooReaderWriter getZooReaderWriter(String string, int timeInMillis, String secret) {
- return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++ return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
+ }
+
+ /**
+ * Gets a reader/writer, retrieving ZooKeeper information from the site configuration. The same instance may be returned for multiple calls.
+ *
+ * @return reader/writer
+ */
+ public IZooReaderWriter getInstance() {
+ synchronized (ZooReaderWriterFactory.class) {
+ if (instance == null) {
+ AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ instance = getZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
+ conf.get(Property.INSTANCE_SECRET));
+ }
+ return instance;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index e7f42af,0000000..3aed7ef
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@@ -1,159 -1,0 +1,161 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class NamespaceConfigurationTest {
+ private static final String NSID = "namespace";
+ private static final String ZOOKEEPERS = "localhost";
+ private static final int ZK_SESSION_TIMEOUT = 120000;
+
+ private String iid;
+ private Instance instance;
+ private AccumuloConfiguration parent;
+ private ZooCacheFactory zcf;
+ private ZooCache zc;
+ private NamespaceConfiguration c;
+
+ @Before
+ public void setUp() {
+ iid = UUID.randomUUID().toString();
+ instance = createMock(Instance.class);
+ parent = createMock(AccumuloConfiguration.class);
+ c = new NamespaceConfiguration(NSID, instance, parent);
+ zcf = createMock(ZooCacheFactory.class);
+ c.setZooCacheFactory(zcf);
+
+ expect(instance.getInstanceID()).andReturn(iid);
+ expectLastCall().anyTimes();
+ expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
+ expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
+ replay(instance);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
+ replay(zcf);
+ }
+
+ @Test
+ public void testGetters() {
+ assertEquals(NSID, c.getNamespaceId());
+ assertEquals(parent, c.getParentConfiguration());
+ }
+
+ @Test
+ public void testGet_InZK() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
- "sekrit".getBytes(Constants.UTF8));
++ "sekrit".getBytes(UTF_8));
+ replay(zc);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGet_InParent() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(null);
+ replay(zc);
+ expect(parent.get(p)).andReturn("sekrit");
+ replay(parent);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGet_SkipParentIfAccumuloNS() {
+ c = new NamespaceConfiguration(Namespaces.ACCUMULO_NAMESPACE_ID, instance, parent);
+ c.setZooCacheFactory(zcf);
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + Namespaces.ACCUMULO_NAMESPACE_ID + Constants.ZNAMESPACE_CONF + "/" + p.getKey()))
+ .andReturn(null);
+ replay(zc);
+ assertNull(c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGetProperties() {
+ PropertyFilter filter = new AllFilter();
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ parent.getProperties(props, filter);
+ replay(parent);
+ List<String> children = new java.util.ArrayList<String>();
+ children.add("foo");
+ children.add("ding");
+ expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF)).andReturn(children);
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "foo")).andReturn(
- "bar".getBytes(Constants.UTF8));
++ "bar".getBytes(UTF_8));
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "ding")).andReturn(
- "dong".getBytes(Constants.UTF8));
++ "dong".getBytes(UTF_8));
+ replay(zc);
+ c.getProperties(props, filter);
+ assertEquals(2, props.size());
+ assertEquals("bar", props.get("foo"));
+ assertEquals("dong", props.get("ding"));
+ }
+
+ @Test
+ public void testObserver() {
+ ConfigurationObserver o = createMock(ConfigurationObserver.class);
+ c.addObserver(o);
+ Collection<ConfigurationObserver> os = c.getObservers();
+ assertEquals(1, os.size());
+ assertTrue(os.contains(o));
+ c.removeObserver(o);
+ os = c.getObservers();
+ assertEquals(0, os.size());
+ }
+
+ @Test
+ public void testInvalidateCache() {
+ // need to do a get so the accessor is created
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
- "sekrit".getBytes(Constants.UTF8));
++ "sekrit".getBytes(UTF_8));
+ zc.clear();
+ replay(zc);
+ c.get(Property.INSTANCE_SECRET);
+ c.invalidateCache();
+ verify(zc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
index 7f07d94,0000000..81148a0
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
@@@ -1,134 -1,0 +1,134 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.endsWith;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ServerConfigurationFactoryTest {
+ private static final String ZK_HOST = "localhost";
+ private static final int ZK_TIMEOUT = 120000;
+ private static final String IID = "iid";
+
+ // use the same mock ZooCacheFactory and ZooCache for all tests
+ private static ZooCacheFactory zcf;
+ private static ZooCache zc;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ zcf = createMock(ZooCacheFactory.class);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache(eq(ZK_HOST), eq(ZK_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
+ expectLastCall().anyTimes();
+ expect(zcf.getZooCache(ZK_HOST, ZK_TIMEOUT)).andReturn(zc);
+ expectLastCall().anyTimes();
+ replay(zcf);
+
+ expect(zc.getChildren(anyObject(String.class))).andReturn(null);
+ expectLastCall().anyTimes();
+ // ConfigSanityCheck looks at timeout
- expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(Constants.UTF8));
++ expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(UTF_8));
+ replay(zc);
+ }
+
+ private Instance instance;
+ private ServerConfigurationFactory scf;
+
+ @Before
+ public void setUp() throws Exception {
+ instance = createMock(Instance.class);
+ expect(instance.getInstanceID()).andReturn(IID);
+ expectLastCall().anyTimes();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ServerConfigurationFactory.clearCachedConfigurations();
+ }
+
+ private void mockInstanceForConfig() {
+ expect(instance.getZooKeepers()).andReturn(ZK_HOST);
+ expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_TIMEOUT);
+ }
+
+ private void ready() {
+ replay(instance);
+ scf = new ServerConfigurationFactory(instance);
+ scf.setZooCacheFactory(zcf);
+ }
+
+ @Test
+ public void testGetInstance() {
+ ready();
+ assertSame(instance, scf.getInstance());
+ }
+
+ @Test
+ public void testGetDefaultConfiguration() {
+ ready();
+ DefaultConfiguration c = scf.getDefaultConfiguration();
+ assertNotNull(c);
+ }
+
+ @Test
+ public void testGetSiteConfiguration() {
+ ready();
+ SiteConfiguration c = scf.getSiteConfiguration();
+ assertNotNull(c);
+ }
+
+ @Test
+ public void testGetConfiguration() {
+ mockInstanceForConfig();
+ ready();
+ AccumuloConfiguration c = scf.getConfiguration();
+ assertNotNull(c);
+ }
+
+ private static final String NSID = "NAMESPACE";
+
+ @Test
+ public void testGetNamespaceConfiguration() {
+ mockInstanceForConfig();
+ ready();
+ NamespaceConfiguration c = scf.getNamespaceConfiguration(NSID);
+ assertEquals(NSID, c.getNamespaceId());
+
+ assertSame(c, scf.getNamespaceConfiguration(NSID));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
index 92580f3,0000000..2f38e4c
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
@@@ -1,145 -1,0 +1,146 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TableConfigurationTest {
+ private static final String TID = "table";
+ private static final String ZOOKEEPERS = "localhost";
+ private static final int ZK_SESSION_TIMEOUT = 120000;
+
+ private String iid;
+ private Instance instance;
+ private NamespaceConfiguration parent;
+ private ZooCacheFactory zcf;
+ private ZooCache zc;
+ private TableConfiguration c;
+
+ @Before
+ public void setUp() {
+ iid = UUID.randomUUID().toString();
+ instance = createMock(Instance.class);
+ parent = createMock(NamespaceConfiguration.class);
+ c = new TableConfiguration(iid, instance, TID, parent);
+ zcf = createMock(ZooCacheFactory.class);
+ c.setZooCacheFactory(zcf);
+
+ expect(instance.getInstanceID()).andReturn(iid);
+ expectLastCall().anyTimes();
+ expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
+ expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
+ replay(instance);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(TableConfWatcher.class))).andReturn(zc);
+ replay(zcf);
+ }
+
+ @Test
+ public void testGetters() {
+ assertEquals(TID, c.getTableId());
+ assertEquals(parent, c.getParentConfiguration());
+ }
+
+ @Test
+ public void testGet_InZK() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
- .andReturn("sekrit".getBytes(Constants.UTF8));
++ .andReturn("sekrit".getBytes(UTF_8));
+ replay(zc);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGet_InParent() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(null);
+ replay(zc);
+ expect(parent.get(p)).andReturn("sekrit");
+ replay(parent);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGetProperties() {
+ PropertyFilter filter = new AllFilter();
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ parent.getProperties(props, filter);
+ replay(parent);
+ List<String> children = new java.util.ArrayList<String>();
+ children.add("foo");
+ children.add("ding");
+ expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF)).andReturn(children);
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(Constants.UTF8));
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(Constants.UTF8));
++ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(UTF_8));
++ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(UTF_8));
+ replay(zc);
+ c.getProperties(props, filter);
+ assertEquals(2, props.size());
+ assertEquals("bar", props.get("foo"));
+ assertEquals("dong", props.get("ding"));
+ }
+
+ @Test
+ public void testObserver() {
+ ConfigurationObserver o = createMock(ConfigurationObserver.class);
+ c.addObserver(o);
+ Collection<ConfigurationObserver> os = c.getObservers();
+ assertEquals(1, os.size());
+ assertTrue(os.contains(o));
+ c.removeObserver(o);
+ os = c.getObservers();
+ assertEquals(0, os.size());
+ }
+
+ @Test
+ public void testInvalidateCache() {
+ // need to do a get so the accessor is created
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
- .andReturn("sekrit".getBytes(Constants.UTF8));
++ .andReturn("sekrit".getBytes(UTF_8));
+ zc.clear();
+ replay(zc);
+ c.get(Property.INSTANCE_SECRET);
+ c.invalidateCache();
+ verify(zc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
index 1b9d8f7,0000000..4369119
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
@@@ -1,186 -1,0 +1,186 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ZooCachePropertyAccessorTest {
+ private static final String PATH = "/root/path/to/props";
+ private static final Property PROP = Property.INSTANCE_SECRET;
+ private static final String KEY = PROP.getKey();
+ private static final String FULL_PATH = PATH + "/" + KEY;
+ private static final String VALUE = "value";
- private static final byte[] VALUE_BYTES = VALUE.getBytes(Constants.UTF8);
++ private static final byte[] VALUE_BYTES = VALUE.getBytes(UTF_8);
+
+ private ZooCache zc;
+ private ZooCachePropertyAccessor a;
+
+ @Before
+ public void setUp() {
+ zc = createMock(ZooCache.class);
+ a = new ZooCachePropertyAccessor(zc);
+ }
+
+ @Test
+ public void testGetter() {
+ assertSame(zc, a.getZooCache());
+ }
+
+ @Test
+ public void testGet_Valid() {
+ expect(zc.get(FULL_PATH)).andReturn(VALUE_BYTES);
+ replay(zc);
+ assertEquals(VALUE, a.get(PROP, PATH, null));
+ }
+
+ @Test
+ public void testGet_Parent() {
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ expect(parent.get(PROP)).andReturn(VALUE);
+ replay(parent);
+ expect(zc.get(FULL_PATH)).andReturn(null);
+ replay(zc);
+ assertEquals(VALUE, a.get(PROP, PATH, parent));
+ }
+
+ @Test
+ public void testGet_Parent_Null() {
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ expect(parent.get(PROP)).andReturn(null);
+ replay(parent);
+ expect(zc.get(FULL_PATH)).andReturn(null);
+ replay(zc);
+ assertNull(a.get(PROP, PATH, parent));
+ }
+
+ @Test
+ public void testGet_Null_NoParent() {
+ expect(zc.get(FULL_PATH)).andReturn(null);
+ replay(zc);
+ assertNull(a.get(PROP, PATH, null));
+ }
+
+ @Test
+ public void testGet_InvalidFormat() {
+ Property badProp = Property.MASTER_CLIENTPORT;
+ expect(zc.get(PATH + "/" + badProp.getKey())).andReturn(VALUE_BYTES);
+ replay(zc);
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ expect(parent.get(badProp)).andReturn("12345");
+ replay(parent);
+ assertEquals("12345", a.get(badProp, PATH, parent));
+ }
+
+ @Test
+ public void testGetProperties() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ parent.getProperties(props, filter);
+ replay(parent);
+ String child1 = "child1";
+ String child2 = "child2";
+ List<String> children = new java.util.ArrayList<String>();
+ children.add(child1);
+ children.add(child2);
+ expect(zc.getChildren(PATH)).andReturn(children);
+ expect(zc.get(PATH + "/" + child1)).andReturn(VALUE_BYTES);
+ expect(zc.get(PATH + "/" + child2)).andReturn(null);
+ replay(zc);
+ expect(filter.accept(child1)).andReturn(true);
+ expect(filter.accept(child2)).andReturn(true);
+ replay(filter);
+
+ a.getProperties(props, PATH, filter, parent, null);
+ assertEquals(1, props.size());
+ assertEquals(VALUE, props.get(child1));
+ verify(parent);
+ }
+
+ @Test
+ public void testGetProperties_NoChildren() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ parent.getProperties(props, filter);
+ replay(parent);
+ expect(zc.getChildren(PATH)).andReturn(null);
+ replay(zc);
+
+ a.getProperties(props, PATH, filter, parent, null);
+ assertEquals(0, props.size());
+ }
+
+ @Test
+ public void testGetProperties_Filter() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ parent.getProperties(props, filter);
+ replay(parent);
+ String child1 = "child1";
+ List<String> children = new java.util.ArrayList<String>();
+ children.add(child1);
+ expect(zc.getChildren(PATH)).andReturn(children);
+ replay(zc);
+ expect(filter.accept(child1)).andReturn(false);
+ replay(filter);
+
+ a.getProperties(props, PATH, filter, parent, null);
+ assertEquals(0, props.size());
+ }
+
+ @Test
+ public void testGetProperties_ParentFilter() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ PropertyFilter parentFilter = createMock(PropertyFilter.class);
+ parent.getProperties(props, parentFilter);
+ replay(parent);
+ expect(zc.getChildren(PATH)).andReturn(null);
+ replay(zc);
+
+ a.getProperties(props, PATH, filter, parent, parentFilter);
+ verify(parent);
+ }
+
+ @Test
+ public void testInvalidateCache() {
+ zc.clear();
+ replay(zc);
+ a.invalidateCache();
+ verify(zc);
+ }
+}
[38/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
index cd26743,0000000..8022c14
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
@@@ -1,210 -1,0 +1,211 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.nio.ByteBuffer;
- import java.nio.charset.Charset;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
+
+public class ValueTest {
+ private static final byte[] toBytes(String s) {
- return s.getBytes(Charset.forName("UTF-8"));
++ return s.getBytes(UTF_8);
+ }
+
+ private static final byte[] DATA = toBytes("data");
+ private static final ByteBuffer DATABUFF = ByteBuffer.allocate(DATA.length);
+ static {
+ DATABUFF.put(DATA);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ DATABUFF.rewind();
+ }
+
+ @Test
+ public void testDefault() {
+ Value v = new Value();
+ assertEquals(0, v.get().length);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullBytesConstructor() {
+ new Value((byte[]) null);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullCopyConstructor() {
+ new Value((Value) null);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullByteBufferConstructor() {
+ new Value((ByteBuffer)null);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullSet() {
+ Value v = new Value();
+ v.set(null);
+ }
+
+ @Test
+ public void testByteArray() {
+ Value v = new Value(DATA);
+ assertArrayEquals(DATA, v.get());
+ assertSame(DATA, v.get());
+ }
+
+ @Test
+ public void testByteArrayCopy() {
+ Value v = new Value(DATA, true);
+ assertArrayEquals(DATA, v.get());
+ assertNotSame(DATA, v.get());
+ }
+
+ @Test
+ public void testByteBuffer() {
+ Value v = new Value(DATABUFF);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testByteBufferCopy() {
+ @SuppressWarnings("deprecation")
+ Value v = new Value(DATABUFF, true);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testValueCopy() {
+ Value ov = createMock(Value.class);
+ expect(ov.get()).andReturn(DATA);
+ expect(ov.getSize()).andReturn(4);
+ replay(ov);
+ Value v = new Value(ov);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testByteArrayOffsetLength() {
+ Value v = new Value(DATA, 0, 4);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testSet() {
+ Value v = new Value();
+ v.set(DATA);
+ assertArrayEquals(DATA, v.get());
+ assertSame(DATA, v.get());
+ }
+
+ @Test
+ public void testCopy() {
+ Value v = new Value();
+ v.copy(DATA);
+ assertArrayEquals(DATA, v.get());
+ assertNotSame(DATA, v.get());
+ }
+
+ @Test
+ public void testGetSize() {
+ Value v = new Value(DATA);
+ assertEquals(DATA.length, v.getSize());
+ }
+
+ @Test
+ public void testGetSizeDefault() {
+ Value v = new Value();
+ assertEquals(0, v.getSize());
+ }
+
+ @Test
+ public void testWriteRead() throws Exception {
+ Value v = new Value(DATA);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ v.write(dos);
+ dos.close();
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ Value v2 = new Value();
+ v2.readFields(dis);
+ dis.close();
+ assertArrayEquals(DATA, v2.get());
+ }
+
+ @Test
+ public void testHashCode() {
+ Value v1 = new Value(DATA);
+ Value v2 = new Value(DATA);
+ assertEquals(v1.hashCode(), v2.hashCode());
+ }
+
+ @Test
+ public void testCompareTo() {
+ Value v1 = new Value(DATA);
+ Value v2 = new Value(toBytes("datb"));
+ assertTrue(v1.compareTo(v2) < 0);
+ assertTrue(v2.compareTo(v1) > 0);
+ Value v1a = new Value(DATA);
+ assertTrue(v1.compareTo(v1a) == 0);
+ Value v3 = new Value(toBytes("datc"));
+ assertTrue(v2.compareTo(v3) < 0);
+ assertTrue(v1.compareTo(v3) < 0);
+ }
+
+ @Test
+ public void testEquals() {
+ Value v1 = new Value(DATA);
+ assertTrue(v1.equals(v1));
+ Value v2 = new Value(DATA);
+ assertTrue(v1.equals(v2));
+ assertTrue(v2.equals(v1));
+ Value v3 = new Value(toBytes("datb"));
+ assertFalse(v1.equals(v3));
+ }
+
+ @Test
+ public void testToArray() {
+ List<byte[]> l = new java.util.ArrayList<byte[]>();
+ byte[] one = toBytes("one");
+ byte[] two = toBytes("two");
+ byte[] three = toBytes("three");
+ l.add(one);
+ l.add(two);
+ l.add(three);
+
+ byte[][] a = Value.toArray(l);
+ assertEquals(3, a.length);
+ assertArrayEquals(one, a[0]);
+ assertArrayEquals(two, a[1]);
+ assertArrayEquals(three, a[2]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index 931ff41,6c4e814..2577e48
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@@ -16,19 -16,11 +16,19 @@@
*/
package org.apache.accumulo.core.security;
++import static com.google.common.base.Charsets.UTF_8;
import static org.apache.accumulo.core.security.ColumnVisibility.quote;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.util.Comparator;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.security.ColumnVisibility.Node;
+import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator;
+import org.apache.accumulo.core.security.ColumnVisibility.NodeType;
+import org.apache.hadoop.io.Text;
import org.junit.Test;
public class ColumnVisibilityTest {
@@@ -166,83 -152,4 +166,83 @@@
cv = new ColumnVisibility(quote("五"));
assertEquals("[\"五\"]", cv.toString());
}
+
+ @Test
+ public void testParseTree() {
+ Node node = parse("(W)|(U&V)");
+ assertNode(node, NodeType.OR, 0, 9);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
+ assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
+ }
+
+ @Test
+ public void testParseTreeWithNoChildren() {
+ Node node = parse("ABC");
+ assertNode(node, NodeType.TERM, 0, 3);
+ }
+
+ @Test
+ public void testParseTreeWithTwoChildren() {
+ Node node = parse("ABC|DEF");
+ assertNode(node, NodeType.OR, 0, 7);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
+ assertNode(node.getChildren().get(1), NodeType.TERM, 4, 7);
+ }
+
+ @Test
+ public void testParseTreeWithParenthesesAndTwoChildren() {
+ Node node = parse("(ABC|DEF)");
+ assertNode(node, NodeType.OR, 1, 8);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 1, 4);
+ assertNode(node.getChildren().get(1), NodeType.TERM, 5, 8);
+ }
+
+ @Test
+ public void testParseTreeWithParenthesizedChildren() {
+ Node node = parse("ABC|(DEF&GHI)");
+ assertNode(node, NodeType.OR, 0, 13);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
+ assertNode(node.getChildren().get(1), NodeType.AND, 5, 12);
+ assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 8);
+ assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 9, 12);
+ }
+
+ @Test
+ public void testParseTreeWithMoreParentheses() {
+ Node node = parse("(W)|(U&V)");
+ assertNode(node, NodeType.OR, 0, 9);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
+ assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
+ assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 6);
+ assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 7, 8);
+ }
+
+ @Test
+ public void testEmptyParseTreesAreEqual() {
+ Comparator<Node> comparator = new NodeComparator(new byte[] {});
+ Node empty = new ColumnVisibility().getParseTree();
+ assertEquals(0, comparator.compare(empty, parse("")));
+ }
+
+ @Test
+ public void testParseTreesOrdering() {
- byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(Constants.UTF8);
++ byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(UTF_8);
+ byte[] flattened = new ColumnVisibility(expression).flatten();
+
+ // Convert to String for indexOf convenience
- String flat = new String(flattened, Constants.UTF8);
++ String flat = new String(flattened, UTF_8);
+ assertTrue("shortest expressions sort first", flat.indexOf('e') < flat.indexOf('|'));
+ assertTrue("shortest children sort first", flat.indexOf('b') < flat.indexOf('a'));
+ }
+
+ private Node parse(String s) {
+ ColumnVisibility v = new ColumnVisibility(s);
+ return v.getParseTree();
+ }
+
+ private void assertNode(Node node, NodeType nodeType, int start, int end) {
+ assertEquals(node.type, nodeType);
+ assertEquals(start, node.start);
+ assertEquals(end, node.end);
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
index b925e4a,0000000..a4e9f02
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
@@@ -1,143 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import javax.security.auth.DestroyFailedException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.client.security.tokens.NullToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class CredentialsTest {
+
+ @Test
+ public void testToThrift() throws DestroyFailedException {
+ // verify thrift serialization
+ Credentials creds = new Credentials("test", new PasswordToken("testing"));
+ TCredentials tCreds = creds.toThrift(new MockInstance());
+ assertEquals("test", tCreds.getPrincipal());
+ assertEquals(PasswordToken.class.getName(), tCreds.getTokenClassName());
+ assertArrayEquals(AuthenticationTokenSerializer.serialize(new PasswordToken("testing")), tCreds.getToken());
+
+ // verify that we can't serialize if it's destroyed
+ creds.getToken().destroy();
+ try {
+ creds.toThrift(new MockInstance());
+ fail();
+ } catch (Exception e) {
+ assertTrue(e instanceof RuntimeException);
+ assertTrue(e.getCause() instanceof AccumuloSecurityException);
+ assertTrue(AccumuloSecurityException.class.cast(e.getCause()).getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
+ }
+ }
+
+ @Test
+ public void roundtripThrift() throws DestroyFailedException {
+ Credentials creds = new Credentials("test", new PasswordToken("testing"));
+ TCredentials tCreds = creds.toThrift(new MockInstance());
+ Credentials roundtrip = Credentials.fromThrift(tCreds);
+ assertEquals("Roundtrip through thirft changed credentials equality", creds, roundtrip);
+ }
+
+ @Test
+ public void testMockConnector() throws AccumuloException, DestroyFailedException, AccumuloSecurityException {
+ Instance inst = new MockInstance();
+ Connector rootConnector = inst.getConnector("root", new PasswordToken());
+ PasswordToken testToken = new PasswordToken("testPass");
+ rootConnector.securityOperations().createLocalUser("testUser", testToken);
+
+ assertFalse(testToken.isDestroyed());
+ testToken.destroy();
+ assertTrue(testToken.isDestroyed());
+ try {
+ inst.getConnector("testUser", testToken);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ assertTrue(e.getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
+ }
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ Credentials nullNullCreds = new Credentials(null, null);
+ Credentials abcNullCreds = new Credentials("abc", new NullToken());
+ Credentials cbaNullCreds = new Credentials("cba", new NullToken());
+ Credentials abcBlahCreds = new Credentials("abc", new PasswordToken("blah"));
+
+ // check hash codes
+ assertEquals(0, nullNullCreds.hashCode());
+ assertEquals("abc".hashCode(), abcNullCreds.hashCode());
+ assertEquals(abcNullCreds.hashCode(), abcBlahCreds.hashCode());
+ assertFalse(abcNullCreds.hashCode() == cbaNullCreds.hashCode());
+
+ // identity
+ assertEquals(abcNullCreds, abcNullCreds);
+ assertEquals(new Credentials("abc", new NullToken()), abcNullCreds);
+ // equal, but different token constructors
- assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(Constants.UTF8))), new Credentials("abc", new PasswordToken("abc")));
++ assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(UTF_8))), new Credentials("abc", new PasswordToken("abc")));
+ // test not equals
+ assertFalse(nullNullCreds.equals(abcBlahCreds));
+ assertFalse(nullNullCreds.equals(abcNullCreds));
+ assertFalse(abcNullCreds.equals(abcBlahCreds));
+ }
+
+ @Test
+ public void testCredentialsSerialization() throws AccumuloSecurityException {
- Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(Constants.UTF8)));
++ Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(UTF_8)));
+ String serialized = creds.serialize();
+ Credentials result = Credentials.deserialize(serialized);
+ assertEquals(creds, result);
+ assertEquals("a:b-c", result.getPrincipal());
+ assertEquals(new PasswordToken("d-e-f"), result.getToken());
+
+ Credentials nullNullCreds = new Credentials(null, null);
+ serialized = nullNullCreds.serialize();
+ result = Credentials.deserialize(serialized);
+ assertEquals(null, result.getPrincipal());
+ assertEquals(null, result.getToken());
+ }
+
+ @Test
+ public void testToString() {
+ Credentials creds = new Credentials(null, null);
+ assertEquals(Credentials.class.getName() + ":null:null:<hidden>", creds.toString());
+ creds = new Credentials("", new NullToken());
+ assertEquals(Credentials.class.getName() + "::" + NullToken.class.getName() + ":<hidden>", creds.toString());
+ creds = new Credentials("abc", null);
+ assertEquals(Credentials.class.getName() + ":abc:null:<hidden>", creds.toString());
+ creds = new Credentials("abc", new PasswordToken(""));
+ assertEquals(Credentials.class.getName() + ":abc:" + PasswordToken.class.getName() + ":<hidden>", creds.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index d31a788,0000000..523279a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -1,103 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.Mutation;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class VisibilityConstraintTest {
+
+ VisibilityConstraint vc;
+ Environment env;
+ Mutation mutation;
+
+ static final ColumnVisibility good = new ColumnVisibility("good");
+ static final ColumnVisibility bad = new ColumnVisibility("bad");
+
+ static final String D = "don't care";
+
+ static final List<Short> ENOAUTH = Arrays.asList((short) 2);
+
+ @Before
+ public void setUp() throws Exception {
+ vc = new VisibilityConstraint();
+ mutation = new Mutation("r");
+
- ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
++ ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(UTF_8));
+
+ AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
+ expect(ac.contains(bs)).andReturn(true);
+ replay(ac);
+
+ env = createMock(Environment.class);
+ expect(env.getAuthorizationsContainer()).andReturn(ac);
+ replay(env);
+ }
+
+ @Test
+ public void testNoVisibility() {
+ mutation.put(D, D, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testVisibilityNoAuth() {
+ mutation.put(D, D, bad, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ public void testGoodVisibilityAuth() {
+ mutation.put(D, D, good, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testCachedVisibilities() {
+ mutation.put(D, D, good, "v");
+ mutation.put(D, D, good, "v2");
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testMixedVisibilities() {
+ mutation.put(D, D, bad, D);
+ mutation.put(D, D, good, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ @Ignore
+ public void testMalformedVisibility() {
+ // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index ea64ac7,0000000..9078ef9
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -1,162 -1,0 +1,162 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+
- import org.apache.accumulo.core.Constants;
+import org.junit.Test;
+
+public class BlockedIOStreamTest {
+ @Test
+ public void testLargeBlockIO() throws IOException {
+ writeRead(1024, 2048);
+ }
+
+ private void writeRead(int blockSize, int expectedSize) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, 1);
+
+ String contentString = "My Blocked Content String";
- byte[] content = contentString.getBytes(Constants.UTF8);
++ byte[] content = contentString.getBytes(UTF_8);
+ blockOut.write(content);
+ blockOut.flush();
+
+ String contentString2 = "My Other Blocked Content String";
- byte[] content2 = contentString2.getBytes(Constants.UTF8);
++ byte[] content2 = contentString2.getBytes(UTF_8);
+ blockOut.write(content2);
+ blockOut.flush();
+
+ blockOut.close();
+ byte[] written = baos.toByteArray();
+ assertEquals(expectedSize, written.length);
+
+ ByteArrayInputStream biis = new ByteArrayInputStream(written);
+ BlockedInputStream blockIn = new BlockedInputStream(biis, blockSize, blockSize);
+ DataInputStream dIn = new DataInputStream(blockIn);
+
+ dIn.readFully(content, 0, content.length);
- String readContentString = new String(content, Constants.UTF8);
++ String readContentString = new String(content, UTF_8);
+
+ assertEquals(contentString, readContentString);
+
+ dIn.readFully(content2, 0, content2.length);
- String readContentString2 = new String(content2, Constants.UTF8);
++ String readContentString2 = new String(content2, UTF_8);
+
+ assertEquals(contentString2, readContentString2);
+
+ blockIn.close();
+ }
+
+ @Test
+ public void testSmallBufferBlockedIO() throws IOException {
+ writeRead(16, (12 + 4) * (int) (Math.ceil(25.0 / 12) + Math.ceil(31.0 / 12)));
+ }
+
+ @Test
+ public void testSpillingOverOutputStream() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ // buffer will be size 12
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, 16, 16);
+ Random r = new Random(22);
+
+ byte[] undersized = new byte[11];
+ byte[] perfectSized = new byte[12];
+ byte[] overSized = new byte[13];
+ byte[] perfectlyOversized = new byte[13];
+ byte filler = (byte) r.nextInt();
+
+ r.nextBytes(undersized);
+ r.nextBytes(perfectSized);
+ r.nextBytes(overSized);
+ r.nextBytes(perfectlyOversized);
+
+ // 1 block
+ blockOut.write(undersized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ // 2 blocks
+ blockOut.write(perfectSized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ // 2 blocks
+ blockOut.write(overSized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ // 3 blocks
+ blockOut.write(undersized);
+ blockOut.write(perfectlyOversized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ blockOut.close();
+ assertEquals(16 * 8, baos.toByteArray().length);
+ }
+
+ @Test
+ public void testGiantWrite() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int blockSize = 16;
+ // buffer will be size 12
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, blockSize);
+ Random r = new Random(22);
+
+ int size = 1024 * 1024 * 128;
+ byte[] giant = new byte[size];
+ byte[] pattern = new byte[1024];
+
+ r.nextBytes(pattern);
+
+ for (int i = 0; i < size / 1024; i++) {
+ System.arraycopy(pattern, 0, giant, i * 1024, 1024);
+ }
+
+ blockOut.write(giant);
+ blockOut.flush();
+
+ blockOut.close();
+ baos.close();
+
+ int blocks = (int) Math.ceil(size / (blockSize - 4.0));
+ byte[] byteStream = baos.toByteArray();
+
+ assertEquals(blocks * 16, byteStream.length);
+
+ DataInputStream blockIn = new DataInputStream(new BlockedInputStream(new ByteArrayInputStream(byteStream), blockSize, blockSize));
+ Arrays.fill(giant, (byte) 0);
+ blockIn.readFully(giant, 0, size);
+ blockIn.close();
+
+ for (int i = 0; i < size / 1024; i++) {
+ byte[] readChunk = new byte[1024];
+ System.arraycopy(giant, i * 1024, readChunk, 0, 1024);
+ assertArrayEquals(pattern, readChunk);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
index 36def05,0000000..a2e4f7a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
@@@ -1,176 -1,0 +1,176 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.format;
+
- import static org.apache.accumulo.core.Constants.UTF8;
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+
+import jline.UnsupportedTerminal;
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DeleterFormatterTest {
+ DeleterFormatter formatter;
+ Map<Key,Value> data;
+ BatchWriter writer;
+ BatchWriter exceptionWriter;
+ Shell shellState;
+
+ ByteArrayOutputStream baos;
+ ConsoleReader reader;
+
+ SettableInputStream input;
+
+ class SettableInputStream extends InputStream {
+ ByteArrayInputStream bais;
+
+ @Override
+ public int read() throws IOException {
+ return bais.read();
+ }
+
+ public void set(String in) {
- bais = new ByteArrayInputStream(in.getBytes(UTF8));
++ bais = new ByteArrayInputStream(in.getBytes(UTF_8));
+ }
+ };
+
+ @Before
+ public void setUp() throws IOException, MutationsRejectedException {
+ input = new SettableInputStream();
+ baos = new ByteArrayOutputStream();
+
+ MutationsRejectedException mre = createMock(MutationsRejectedException.class);
+
+ writer = createNiceMock(BatchWriter.class);
+ exceptionWriter = createNiceMock(BatchWriter.class);
+ exceptionWriter.close();
+ expectLastCall().andThrow(mre);
+ exceptionWriter.addMutation(anyObject(Mutation.class));
+ expectLastCall().andThrow(mre);
+
+ shellState = createNiceMock(Shell.class);
+
+ reader = new ConsoleReader(input, baos, new UnsupportedTerminal());
+ expect(shellState.getReader()).andReturn(reader).anyTimes();
+
+ replay(writer, exceptionWriter, shellState);
+
+ data = new TreeMap<Key,Value>();
- data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF8)));
++ data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF_8)));
+ }
+
+ @Test
+ public void testEmpty() {
+ formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true);
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testSingle() throws IOException {
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[DELETED]", " r ", "cf", "cq", "value");
+ }
+
+ @Test
+ public void testNo() throws IOException {
+ input.set("no\n");
- data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[SKIPPED]", " r ", "cf", "cq", "value");
+
+ assertTrue(formatter.hasNext());
+ }
+
+ @Test
+ public void testNoConfirmation() throws IOException {
+ input.set("");
- data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[SKIPPED]", " r ", "cf", "cq", "value");
+
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testYes() throws IOException {
+ input.set("y\nyes\n");
- data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ verify("[DELETED]", " r ", "cf", "cq", "value");
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ verify("[DELETED]", " z ", "v2");
+ }
+
+ @Test
+ public void testMutationException() {
+ formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ assertFalse(formatter.hasNext());
+ }
+
+ private void verify(String... chunks) throws IOException {
+ reader.flush();
+
+ String output = baos.toString();
+ for (String chunk : chunks) {
+ assertTrue(output.contains(chunk));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 8fc68fd,0000000..eef06c3
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@@ -1,90 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.format;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ShardedTableDistributionFormatterTest {
+ ShardedTableDistributionFormatter formatter;
+
+ Map<Key,Value> data;
+
+ @Before
+ public void setUp() {
+ data = new TreeMap<Key,Value>();
+ formatter = new ShardedTableDistributionFormatter();
+ }
+
+ @Test
+ public void testInitialize() {
+ data.put(new Key(), new Value());
+ data.put(new Key("r", "~tab"), new Value());
+ formatter.initialize(data.entrySet(), false);
+
+ assertTrue(formatter.hasNext());
+ formatter.next();
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testAggregate() {
- data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(Constants.UTF8)));
- data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(Constants.UTF8)));
- data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(Constants.UTF8)));
++ data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(UTF_8)));
++ data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(UTF_8)));
++ data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(UTF_8)));
+
+ formatter.initialize(data.entrySet(), false);
+
+ String[] resultLines = formatter.next().split("\n");
+ List<String> results = Arrays.asList(resultLines).subList(2, 4);
+
+ assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("NULL", 1)));
+ assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("19700101", 2)));
+
+ assertFalse(formatter.hasNext());
+ }
+
+ private static class AggregateReportChecker implements Predicate {
+ private String day;
+ private int count;
+
+ AggregateReportChecker(String day, int count) {
+ this.day = day;
+ this.count = count;
+ }
+
+ @Override
+ public boolean evaluate(Object arg) {
+ String resLine = (String) arg;
+ return resLine.startsWith(this.day) && resLine.endsWith("" + this.count);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 1f8c32d,e411d22..af31cf2
--- a/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@@ -16,11 -16,10 +16,12 @@@
*/
package org.apache.accumulo.fate;
- import java.nio.charset.Charset;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@@ -37,34 -35,8 +38,33 @@@ import org.apache.zookeeper.KeeperExcep
* A utility to administer FATE operations
*/
public class AdminUtil<T> {
- private static final Charset UTF8 = Charset.forName("UTF-8");
+ private boolean exitOnError = false;
+
+ /**
+ * Default constructor
+ */
+ public AdminUtil() {
+ this(true);
+ }
+
+ /**
+ * Constructor
+ *
+ * @param exitOnError
+ * <code>System.exit(1)</code> on error if true
+ */
+ public AdminUtil(boolean exitOnError) {
+ super();
+ this.exitOnError = exitOnError;
+ }
+
public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath) throws KeeperException, InterruptedException {
+ print(zs, zk, lockPath, new Formatter(System.out), null, null);
+ }
+
+ public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath, Formatter fmt, Set<Long> filterTxid, EnumSet<TStatus> filterStatus)
+ throws KeeperException, InterruptedException {
Map<Long,List<String>> heldLocks = new HashMap<Long,List<String>>();
Map<Long,List<String>> waitingLocks = new HashMap<Long,List<String>>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index b6890c6,3ef6b0a..ff35364
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -32,8 -33,7 +33,7 @@@ import org.apache.zookeeper.ZooKeeper
import org.apache.zookeeper.ZooKeeper.States;
public class ZooSession {
- private static final Charset UTF8 = Charset.forName("UTF-8");
-
+
public static class ZooSessionShutdownException extends RuntimeException {
private static final long serialVersionUID = 1L;
@@@ -46,19 -46,18 +46,19 @@@
public ZooSessionInfo(ZooKeeper zooKeeper, ZooWatcher watcher) {
this.zooKeeper = zooKeeper;
}
-
+
ZooKeeper zooKeeper;
}
-
+
private static Map<String,ZooSessionInfo> sessions = new HashMap<String,ZooSessionInfo>();
-
+
private static String sessionKey(String keepers, int timeout, String scheme, byte[] auth) {
- return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF8));
+ return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF_8));
}
-
+
private static class ZooWatcher implements Watcher {
-
+
+ @Override
public void process(WatchedEvent event) {
if (event.getState() == KeeperState.Expired) {
log.debug("Session expired, state of current session : " + event.getState());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index d5dc1e9,0000000..8216441
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -1,851 -1,0 +1,853 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.cluster.AccumuloCluster;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.gc.SimpleGarbageCollector;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.AccumuloStatus;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
+import org.apache.accumulo.start.Main;
+import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.commons.configuration.MapConfiguration;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Maps;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than {@link org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower.
+ *
+ * @since 1.6.0
+ */
+public class MiniAccumuloClusterImpl implements AccumuloCluster {
+ private static final Logger log = LoggerFactory.getLogger(MiniAccumuloClusterImpl.class);
+
+ public static class LogWriter extends Daemon {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {}
+ }
+ }
+
+ private boolean initialized = false;
+ private Process zooKeeperProcess = null;
+ private Process masterProcess = null;
+ private Process gcProcess = null;
+ private List<Process> tabletServerProcesses = Collections.synchronizedList(new ArrayList<Process>());
+
+ private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
+
+ private File zooCfgFile;
+ private String dfsUri;
+
+ public List<LogWriter> getLogWriters() {
+ return logWriters;
+ }
+
+ private List<LogWriter> logWriters = new ArrayList<MiniAccumuloClusterImpl.LogWriter>();
+
+ private MiniAccumuloConfigImpl config;
+ private MiniDFSCluster miniDFS = null;
+ private List<Process> cleanup = new ArrayList<Process>();
+
+ private ExecutorService executor;
+
+ public Process exec(Class<?> clazz, String... args) throws IOException {
+ return exec(clazz, null, args);
+ }
+
+ public Process exec(Class<?> clazz, List<String> jvmArgs, String... args) throws IOException {
+ ArrayList<String> jvmArgs2 = new ArrayList<String>(1 + (jvmArgs == null ? 0 : jvmArgs.size()));
+ jvmArgs2.add("-Xmx" + config.getDefaultMemory());
+ if (jvmArgs != null)
+ jvmArgs2.addAll(jvmArgs);
+ Process proc = _exec(clazz, jvmArgs2, args);
+ cleanup.add(proc);
+ return proc;
+ }
+
+ private boolean containsSiteFile(File f) {
+ return f.isDirectory() && f.listFiles(new FileFilter() {
+
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().endsWith("site.xml");
+ }
+ }).length > 0;
+ }
+
+ private void append(StringBuilder classpathBuilder, URL url) throws URISyntaxException {
+ File file = new File(url.toURI());
+ // do not include dirs containing hadoop or accumulo site files
+ if (!containsSiteFile(file))
+ classpathBuilder.append(File.pathSeparator).append(file.getAbsolutePath());
+ }
+
+ private String getClasspath() throws IOException {
+
+ try {
+ ArrayList<ClassLoader> classloaders = new ArrayList<ClassLoader>();
+
+ ClassLoader cl = this.getClass().getClassLoader();
+
+ while (cl != null) {
+ classloaders.add(cl);
+ cl = cl.getParent();
+ }
+
+ Collections.reverse(classloaders);
+
+ StringBuilder classpathBuilder = new StringBuilder();
+ classpathBuilder.append(config.getConfDir().getAbsolutePath());
+
+ if (config.getHadoopConfDir() != null)
+ classpathBuilder.append(File.pathSeparator).append(config.getHadoopConfDir().getAbsolutePath());
+
+ if (config.getClasspathItems() == null) {
+
+ // assume 0 is the system classloader and skip it
+ for (int i = 1; i < classloaders.size(); i++) {
+ ClassLoader classLoader = classloaders.get(i);
+
+ if (classLoader instanceof URLClassLoader) {
+
+ URLClassLoader ucl = (URLClassLoader) classLoader;
+
+ for (URL u : ucl.getURLs()) {
+ append(classpathBuilder, u);
+ }
+
+ } else if (classLoader instanceof VFSClassLoader) {
+
+ VFSClassLoader vcl = (VFSClassLoader) classLoader;
+ for (FileObject f : vcl.getFileObjects()) {
+ append(classpathBuilder, f.getURL());
+ }
+ } else {
+ throw new IllegalArgumentException("Unknown classloader type : " + classLoader.getClass().getName());
+ }
+ }
+ } else {
+ for (String s : config.getClasspathItems())
+ classpathBuilder.append(File.pathSeparator).append(s);
+ }
+
+ return classpathBuilder.toString();
+
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private Process _exec(Class<?> clazz, List<String> extraJvmOpts, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = getClasspath();
+
+ String className = clazz.getName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+ argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath));
+ argList.addAll(extraJvmOpts);
+ for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) {
+ argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue()));
+ }
+ argList.addAll(Arrays.asList("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-Dapple.awt.UIElement=true", Main.class.getName(),
+ className));
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath());
+ String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator);
+ builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath);
+ builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath);
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+ builder.environment().put("ACCUMULO_CONF_DIR", config.getConfDir().getAbsolutePath());
+ // hadoop-2.2 puts error messages in the logs if this is not set
+ builder.environment().put("HADOOP_HOME", config.getDir().getAbsolutePath());
+ if (config.getHadoopConfDir() != null)
+ builder.environment().put("HADOOP_CONF_DIR", config.getHadoopConfDir().getAbsolutePath());
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private Process _exec(Class<?> clazz, ServerType serverType, String... args) throws IOException {
+
+ List<String> jvmOpts = new ArrayList<String>();
+ jvmOpts.add("-Xmx" + config.getMemory(serverType));
+
+ if (config.isJDWPEnabled()) {
+ Integer port = PortUtils.getRandomFreePort();
+ jvmOpts.addAll(buildRemoteDebugParams(port));
+ debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
+ }
+ return _exec(clazz, jvmOpts, args);
+ }
+
+ /**
+ *
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ */
+ public MiniAccumuloClusterImpl(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfigImpl(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ */
+ @SuppressWarnings("deprecation")
+ public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException {
+
+ this.config = config.initialize();
+
+ config.getConfDir().mkdirs();
+ config.getLogDir().mkdirs();
+ config.getLibDir().mkdirs();
+ config.getLibExtDir().mkdirs();
+
+ if (!config.useExistingInstance()) {
+ config.getZooKeeperDir().mkdirs();
+ config.getWalogDir().mkdirs();
+ config.getAccumuloDir().mkdirs();
+ }
+
+ if (config.useMiniDFS()) {
+ File nn = new File(config.getAccumuloDir(), "nn");
+ nn.mkdirs();
+ File dn = new File(config.getAccumuloDir(), "dn");
+ dn.mkdirs();
+ File dfs = new File(config.getAccumuloDir(), "dfs");
+ dfs.mkdirs();
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
+ conf.set("dfs.support.append", "true");
+ conf.set("dfs.datanode.synconclose", "true");
+ conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
+ String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath());
+ miniDFS = new MiniDFSCluster(conf, 1, true, null);
+ if (oldTestBuildData == null)
+ System.clearProperty("test.build.data");
+ else
+ System.setProperty("test.build.data", oldTestBuildData);
+ miniDFS.waitClusterUp();
+ InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
+ dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
+ File coreFile = new File(config.getConfDir(), "core-site.xml");
+ writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
+ File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
+ writeConfig(hdfsFile, conf);
+
+ Map<String,String> siteConfig = config.getSiteConfig();
+ siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
+ siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
+ config.setSiteConfig(siteConfig);
+ } else if (config.useExistingInstance()) {
+ dfsUri = CachedConfiguration.getInstance().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
+ } else {
+ dfsUri = "file://";
+ }
+
+ File clientConfFile = config.getClientConfFile();
+ // Write only the properties that correspond to ClientConfiguration properties
+ writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate<Entry<String,String>>() {
+ @Override
+ public boolean apply(Entry<String,String> v) {
+ return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null;
+ }
+ }));
+
+ File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
+ writeConfig(siteFile, config.getSiteConfig().entrySet());
+
+ if (!config.useExistingInstance()) {
+ zooCfgFile = new File(config.getConfDir(), "zoo.cfg");
+ FileWriter fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "2000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", config.getZooKeeperPort() + "");
+ zooCfg.setProperty("maxClientCnxns", "1000");
+ zooCfg.setProperty("dataDir", config.getZooKeeperDir().getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ // disable audit logging for mini....
+ InputStream auditStream = this.getClass().getResourceAsStream("/auditLog.xml");
+
+ if (auditStream != null) {
+ FileUtils.copyInputStreamToFile(auditStream, new File(config.getConfDir(), "auditLog.xml"));
+ }
+ }
+
+ private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings) throws IOException {
+ FileWriter fileWriter = new FileWriter(file);
+ fileWriter.append("<configuration>\n");
+
+ for (Entry<String,String> entry : settings) {
+ String value = entry.getValue().replace("&", "&").replace("<", "<").replace(">", ">");
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + value + "</value></property>\n");
+ }
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+ }
+
+ private void writeConfigProperties(File file, Map<String,String> settings) throws IOException {
+ FileWriter fileWriter = new FileWriter(file);
+
+ for (Entry<String,String> entry : settings.entrySet())
+ fileWriter.append(entry.getKey() + "=" + entry.getValue() + "\n");
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IllegalStateException
+ * if already started
+ */
+ @Override
+ public synchronized void start() throws IOException, InterruptedException {
+ if (config.useExistingInstance()) {
+ Configuration acuConf = config.getAccumuloConfiguration();
+ Configuration hadoopConf = config.getHadoopConfiguration();
+
+ ConfigurationCopy cc = new ConfigurationCopy(acuConf);
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get(cc, hadoopConf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
+ IZooReaderWriter zrw = new ZooReaderWriterFactory().getZooReaderWriter(cc.get(Property.INSTANCE_ZK_HOST),
+ (int) cc.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), cc.get(Property.INSTANCE_SECRET));
+
+ String rootPath = ZooUtil.getRoot(instanceIdFromFile);
+
+ String instanceName = null;
+ try {
+ for (String name : zrw.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ byte[] bytes = zrw.getData(instanceNamePath, new Stat());
- String iid = new String(bytes, Constants.UTF8);
++ String iid = new String(bytes, UTF_8);
+ if (iid.equals(instanceIdFromFile)) {
+ instanceName = name;
+ }
+ }
+ } catch (KeeperException e) {
+ throw new RuntimeException("Unable to read instance name from zookeeper.", e);
+ }
+ if (instanceName == null)
+ throw new RuntimeException("Unable to read instance name from zookeeper.");
+
+ config.setInstanceName(instanceName);
+ if (!AccumuloStatus.isAccumuloOffline(zrw, rootPath))
+ throw new RuntimeException("The Accumulo instance being used is already running. Aborting.");
+ } else {
+ if (!initialized) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloClusterImpl.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ if (zooKeeperProcess == null) {
+ zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
+ }
+
+ if (!initialized) {
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ long startTime = System.currentTimeMillis();
+ while (true) {
+ Socket s = null;
+ try {
+ s = new Socket("localhost", config.getZooKeeperPort());
+ s.getOutputStream().write("ruok\n".getBytes());
+ s.getOutputStream().flush();
+ byte buffer[] = new byte[100];
+ int n = s.getInputStream().read(buffer);
+ if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
+ break;
+ } catch (Exception e) {
+ if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
+ throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in "
+ + config.getLogDir() + " for errors. Last exception: " + e);
+ }
+ } finally {
+ if (s != null)
+ s.close();
+ }
+ }
+ Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ initialized = true;
+ }
+ }
+
+ log.info("Starting MAC against instance {} and zookeeper(s) {}.", config.getInstanceName(), config.getZooKeepers());
+
+ synchronized (tabletServerProcesses) {
+ for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
+ tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
+ }
+ }
+ int ret = 0;
+ for (int i = 0; i < 5; i++) {
+ ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()).waitFor();
+ if (ret == 0)
+ break;
+ UtilWaitThread.sleep(1000);
+ }
+ if (ret != 0) {
+ throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ if (masterProcess == null) {
+ masterProcess = _exec(Master.class, ServerType.MASTER);
+ }
+
+ if (gcProcess == null) {
+ gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+ }
+
+ if (null == executor) {
+ executor = Executors.newSingleThreadExecutor();
+ }
+ }
+
+ private List<String> buildRemoteDebugParams(int port) {
+ return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d", port)});
+ }
+
+ /**
+ * @return generated remote debug ports if in debug mode.
+ * @since 1.6.0
+ */
+ public Set<Pair<ServerType,Integer>> getDebugPorts() {
+ return debugPorts;
+ }
+
+ List<ProcessReference> references(Process... procs) {
+ List<ProcessReference> result = new ArrayList<ProcessReference>();
+ for (Process proc : procs) {
+ result.add(new ProcessReference(proc));
+ }
+ return result;
+ }
+
+ public Map<ServerType,Collection<ProcessReference>> getProcesses() {
+ Map<ServerType,Collection<ProcessReference>> result = new HashMap<ServerType,Collection<ProcessReference>>();
+ result.put(ServerType.MASTER, references(masterProcess));
+ result.put(ServerType.TABLET_SERVER, references(tabletServerProcesses.toArray(new Process[0])));
+ result.put(ServerType.ZOOKEEPER, references(zooKeeperProcess));
+ if (null != gcProcess) {
+ result.put(ServerType.GARBAGE_COLLECTOR, references(gcProcess));
+ }
+ return result;
+ }
+
+ public void killProcess(ServerType type, ProcessReference proc) throws ProcessNotFoundException, InterruptedException {
+ boolean found = false;
+ switch (type) {
+ case MASTER:
+ if (proc.equals(masterProcess)) {
+ masterProcess.destroy();
+ masterProcess.waitFor();
+ masterProcess = null;
+ found = true;
+ }
+ break;
+ case TABLET_SERVER:
+ synchronized (tabletServerProcesses) {
+ for (Process tserver : tabletServerProcesses) {
+ if (proc.equals(tserver)) {
+ tabletServerProcesses.remove(tserver);
+ tserver.destroy();
+ tserver.waitFor();
+ found = true;
+ break;
+ }
+ }
+ }
+ break;
+ case ZOOKEEPER:
+ if (proc.equals(zooKeeperProcess)) {
+ zooKeeperProcess.destroy();
+ zooKeeperProcess.waitFor();
+ zooKeeperProcess = null;
+ found = true;
+ }
+ break;
+ case GARBAGE_COLLECTOR:
+ if (proc.equals(gcProcess)) {
+ gcProcess.destroy();
+ gcProcess.waitFor();
+ gcProcess = null;
+ found = true;
+ }
+ break;
+ }
+ if (!found)
+ throw new ProcessNotFoundException();
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+ @Override
+ public String getInstanceName() {
+ return config.getInstanceName();
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+ @Override
+ public String getZooKeepers() {
+ return config.getZooKeepers();
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. However its probably best to
+ * call stop in a finally block as soon as possible.
+ */
+ @Override
+ public synchronized void stop() throws IOException, InterruptedException {
+ if (null == executor) {
+ // keep repeated calls to stop() from failing
+ return;
+ }
+
+ for (LogWriter lw : logWriters) {
+ lw.flush();
+ }
+
+ if (gcProcess != null) {
+ try {
+ stopProcessWithTimeout(gcProcess, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("GarbageCollector did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("GarbageCollector did not fully stop after 30 seconds", e);
+ }
+ }
+ if (masterProcess != null) {
+ try {
+ stopProcessWithTimeout(masterProcess, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("Master did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("Master did not fully stop after 30 seconds", e);
+ }
+ }
+ if (tabletServerProcesses != null) {
+ synchronized (tabletServerProcesses) {
+ for (Process tserver : tabletServerProcesses) {
+ try {
+ stopProcessWithTimeout(tserver, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("TabletServer did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("TabletServer did not fully stop after 30 seconds", e);
+ }
+ }
+ }
+ }
+ if (zooKeeperProcess != null) {
+ try {
+ stopProcessWithTimeout(zooKeeperProcess, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("ZooKeeper did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("ZooKeeper did not fully stop after 30 seconds", e);
+ }
+ }
+
+ zooKeeperProcess = null;
+ masterProcess = null;
+ gcProcess = null;
+ tabletServerProcesses.clear();
+
+ // ACCUMULO-2985 stop the ExecutorService after we finished using it to stop accumulo procs
+ if (null != executor) {
+ List<Runnable> tasksRemaining = executor.shutdownNow();
+
+ // the single thread executor shouldn't have any pending tasks, but check anyways
+ if (!tasksRemaining.isEmpty()) {
+ log.warn("Unexpectedly had " + tasksRemaining.size() + " task(s) remaining in threadpool for execution when being stopped");
+ }
+
+ executor = null;
+ }
+
+ if (config.useMiniDFS() && miniDFS != null)
+ miniDFS.shutdown();
+ for (Process p : cleanup) {
+ p.destroy();
+ p.waitFor();
+ }
+ miniDFS = null;
+ }
+
+ /**
+ * @since 1.6.0
+ */
+ @Override
+ public MiniAccumuloConfigImpl getConfig() {
+ return config;
+ }
+
+ /**
+ * Utility method to get a connector to the MAC.
+ *
+ * @since 1.6.0
+ */
+ @Override
+ public Connector getConnector(String user, String passwd) throws AccumuloException, AccumuloSecurityException {
+ Instance instance = new ZooKeeperInstance(getClientConfig());
+ return instance.getConnector(user, new PasswordToken(passwd));
+ }
+
+ @Override
+ public ClientConfiguration getClientConfig() {
+ return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(
+ this.getZooKeepers());
+ }
+
+ public FileSystem getFileSystem() {
+ try {
+ return FileSystem.get(new URI(dfsUri), new Configuration());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // Visible for testing
+ protected void setShutdownExecutor(ExecutorService svc) {
+ this.executor = svc;
+ }
+
+ // Visible for testing
+ protected ExecutorService getShutdownExecutor() {
+ return executor;
+ }
+
+ private int stopProcessWithTimeout(final Process proc, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ FutureTask<Integer> future = new FutureTask<Integer>(new Callable<Integer>() {
+ @Override
+ public Integer call() throws InterruptedException {
+ proc.destroy();
+ return proc.waitFor();
+ }
+ });
+
+ executor.execute(future);
+
+ return future.get(timeout, unit);
+ }
+
+ /**
+ * Get programmatic interface to information available in a normal monitor. XXX the returned structure won't contain information about the metadata table
+ * until there is data in it. e.g. if you want to see the metadata table you should create a table.
+ *
+ * @since 1.6.1
+ */
+ public MasterMonitorInfo getMasterMonitorInfo() throws AccumuloException, AccumuloSecurityException {
+ MasterClientService.Iface client = null;
+ MasterMonitorInfo stats = null;
+ try {
+ Instance instance = new ZooKeeperInstance(getClientConfig());
+ client = MasterClient.getConnectionWithRetry(instance);
+ stats = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get(instance).toThrift(instance));
+ } catch (ThriftSecurityException exception) {
+ throw new AccumuloSecurityException(exception);
+ } catch (TException exception) {
+ throw new AccumuloException(exception);
+ } finally {
+ if (client != null) {
+ MasterClient.close(client);
+ }
+ }
+ return stats;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
index be29dbb,48e2906..30da0cba
--- a/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
@@@ -16,6 -16,6 +16,8 @@@
*/
package org.apache.accumulo.proxy;
++import static com.google.common.base.Charsets.UTF_8;
++
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
@@@ -194,13 -172,11 +195,13 @@@ public class ProxyServer implements Acc
}
protected Connector getConnector(ByteBuffer login) throws Exception {
- String[] pair = new String(login.array(), login.position(), login.remaining(), Constants.UTF8).split(",", 2);
- TCredentials user = CredentialHelper.fromByteArray(ByteBufferUtil.toBytes(login));
- if (user == null)
- throw new org.apache.accumulo.proxy.thrift.AccumuloSecurityException("unknown user");
- Connector connector = instance.getConnector(user.getPrincipal(), CredentialHelper.extractToken(user));
- return connector;
++ String[] pair = new String(login.array(), login.position(), login.remaining(), UTF_8).split(",", 2);
+ if (instance.getInstanceID().equals(pair[0])) {
+ Credentials creds = Credentials.deserialize(pair[1]);
+ return instance.getConnector(creds.getPrincipal(), creds.getToken());
+ } else {
+ throw new org.apache.accumulo.core.client.AccumuloSecurityException(pair[0], org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.INVALID_INSTANCEID);
+ }
}
private void handleAccumuloException(AccumuloException e) throws org.apache.accumulo.proxy.thrift.TableNotFoundException,
@@@ -1489,7 -1432,8 +1490,7 @@@
public ByteBuffer login(String principal, Map<String,String> loginProperties) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException {
try {
AuthenticationToken token = getToken(principal, loginProperties);
- ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(Constants.UTF8));
- TCredentials credential = CredentialHelper.create(principal, token, instance.getInstanceID());
- ByteBuffer login = ByteBuffer.wrap(CredentialHelper.asByteArray(credential));
++ ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(UTF_8));
getConnector(login); // check to make sure user exists
return login;
} catch (AccumuloSecurityException e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------
[37/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 1e7658a,0000000..6c40094
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -1,329 -1,0 +1,331 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.Version;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.ReadOnlyStore;
+import org.apache.accumulo.fate.ReadOnlyTStore;
+import org.apache.accumulo.fate.ZooStore;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.watcher.Log4jConfiguration;
+import org.apache.accumulo.server.watcher.MonitorLog4jWatcher;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.LogLog;
+import org.apache.log4j.xml.DOMConfigurator;
+import org.apache.zookeeper.KeeperException;
+
+public class Accumulo {
+
+ private static final Logger log = Logger.getLogger(Accumulo.class);
+
+ public static synchronized void updateAccumuloVersion(VolumeManager fs, int oldVersion) {
+ for (Volume volume : fs.getVolumes()) {
+ try {
+ if (getAccumuloPersistentVersion(fs) == oldVersion) {
+ log.debug("Attempting to upgrade " + volume);
+ Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
+ fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
+ // TODO document failure mode & recovery if FS permissions cause above to work and below to fail ACCUMULO-2596
+ Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(oldVersion));
+ if (!fs.delete(prevDataVersionLoc)) {
+ throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
+ }
+ }
+ }
+
+ public static synchronized int getAccumuloPersistentVersion(FileSystem fs, Path path) {
+ int dataVersion;
+ try {
+ FileStatus[] files = fs.listStatus(path);
+ if (files == null || files.length == 0) {
+ dataVersion = -1; // assume it is 0.5 or earlier
+ } else {
+ dataVersion = Integer.parseInt(files[0].getPath().getName());
+ }
+ return dataVersion;
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to read accumulo version: an error occurred.", e);
+ }
+ }
+
+ public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
+ // It doesn't matter which Volume is used as they should all have the data version stored
+ Volume v = fs.getVolumes().iterator().next();
+ Path path = ServerConstants.getDataVersionLocation(v);
+ return getAccumuloPersistentVersion(v.getFileSystem(), path);
+ }
+
+ public static synchronized Path getAccumuloInstanceIdPath(VolumeManager fs) {
+ // It doesn't matter which Volume is used as they should all have the instance ID stored
+ Volume v = fs.getVolumes().iterator().next();
+ return ServerConstants.getInstanceIdLocation(v);
+ }
+
+ public static void enableTracing(String address, String application) {
+ try {
+ DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address);
+ } catch (Exception ex) {
+ log.error("creating remote sink for trace spans", ex);
+ }
+ }
+
+ /**
+ * Finds the best log4j configuration file. A generic file is used only if an
+ * application-specific file is not available. An XML file is preferred over
+ * a properties file, if possible.
+ *
+ * @param confDir directory where configuration files should reside
+ * @param application application name for configuration file name
+ * @return configuration file name
+ */
+ static String locateLogConfig(String confDir, String application) {
+ String explicitConfigFile = System.getProperty("log4j.configuration");
+ if (explicitConfigFile != null) {
+ return explicitConfigFile;
+ }
+ String[] configFiles = {
+ String.format("%s/%s_logger.xml", confDir, application),
+ String.format("%s/%s_logger.properties", confDir, application),
+ String.format("%s/generic_logger.xml", confDir),
+ String.format("%s/generic_logger.properties", confDir)
+ };
+ String defaultConfigFile = configFiles[2]; // generic_logger.xml
+ for (String f : configFiles) {
+ if (new File(f).exists()) {
+ return f;
+ }
+ }
+ return defaultConfigFile;
+ }
+
+ public static void setupLogging(String application) throws UnknownHostException {
+ System.setProperty("org.apache.accumulo.core.application", application);
+
+ if (System.getenv("ACCUMULO_LOG_DIR") != null)
+ System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_LOG_DIR"));
+ else
+ System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_HOME") + "/logs/");
+
+ String localhost = InetAddress.getLocalHost().getHostName();
+ System.setProperty("org.apache.accumulo.core.ip.localhost.hostname", localhost);
+
+ // Use a specific log config, if it exists
+ String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
+ // Turn off messages about not being able to reach the remote logger... we protect against that.
+ LogLog.setQuietMode(true);
+
+ // Read the auditing config
+ String auditConfig = String.format("%s/auditLog.xml", System.getenv("ACCUMULO_CONF_DIR"));
+
+ // Set up local file-based logging right away
+ Log4jConfiguration logConf = new Log4jConfiguration(logConfigFile);
+ logConf.resetLogger();
+
+ // Watch the auditLog.xml for the future updates
+ DOMConfigurator.configureAndWatch(auditConfig, 5000);
+ }
+
+ public static void init(VolumeManager fs, ServerConfiguration serverConfig, String application) throws IOException {
+ final AccumuloConfiguration conf = serverConfig.getConfiguration();
+ final Instance instance = serverConfig.getInstance();
+
+ // Use a specific log config, if it exists
+ final String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
+
+ // Set up polling log4j updates and log-forwarding using information advertised in zookeeper by the monitor
+ MonitorLog4jWatcher logConfigWatcher = new MonitorLog4jWatcher(instance.getInstanceID(), logConfigFile);
+ logConfigWatcher.setDelay(5000L);
+ logConfigWatcher.start();
+
+ // Makes sure the log-forwarding to the monitor is configured
+ int logPort = conf.getPort(Property.MONITOR_LOG4J_PORT);
+ System.setProperty("org.apache.accumulo.core.host.log.port", Integer.toString(logPort));
+
+ log.info(application + " starting");
+ log.info("Instance " + serverConfig.getInstance().getInstanceID());
+ int dataVersion = Accumulo.getAccumuloPersistentVersion(fs);
+ log.info("Data Version " + dataVersion);
+ Accumulo.waitForZookeeperAndHdfs(fs);
+
+ Version codeVersion = new Version(Constants.VERSION);
+ if (!(canUpgradeFromDataVersion(dataVersion))) {
+ throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible with files stored using data version " + dataVersion);
+ }
+
+ TreeMap<String,String> sortedProps = new TreeMap<String,String>();
+ for (Entry<String,String> entry : conf)
+ sortedProps.put(entry.getKey(), entry.getValue());
+
+ for (Entry<String,String> entry : sortedProps.entrySet()) {
+ String key = entry.getKey();
+ log.info(key + " = " + (Property.isSensitive(key) ? "<hidden>" : entry.getValue()));
+ }
+
+ monitorSwappiness();
+ }
+
+ /**
+ * Sanity check that the current persistent version is allowed to upgrade to the version of Accumulo running.
+ * @param dataVersion the version that is persisted in the backing Volumes
+ */
+ public static boolean canUpgradeFromDataVersion(final int dataVersion) {
+ return dataVersion == ServerConstants.DATA_VERSION || dataVersion == ServerConstants.PREV_DATA_VERSION || dataVersion == ServerConstants.TWO_DATA_VERSIONS_AGO;
+ }
+
+ /**
+ * Does the data version number stored in the backing Volumes indicate we need to upgrade something?
+ */
+ public static boolean persistentVersionNeedsUpgrade(final int accumuloPersistentVersion) {
+ return accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO || accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION;
+ }
+
+ /**
+ *
+ */
+ public static void monitorSwappiness() {
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ String procFile = "/proc/sys/vm/swappiness";
+ File swappiness = new File(procFile);
+ if (swappiness.exists() && swappiness.canRead()) {
+ InputStream is = new FileInputStream(procFile);
+ try {
+ byte[] buffer = new byte[10];
+ int bytes = is.read(buffer);
- String setting = new String(buffer, 0, bytes, Constants.UTF8);
++ String setting = new String(buffer, 0, bytes, UTF_8);
+ setting = setting.trim();
+ if (bytes > 0 && Integer.parseInt(setting) > 10) {
+ log.warn("System swappiness setting is greater than ten (" + setting + ") which can cause time-sensitive operations to be delayed. "
+ + " Accumulo is time sensitive because it needs to maintain distributed lock agreement.");
+ }
+ } finally {
+ is.close();
+ }
+ }
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+ }
+ }, 1000, 10 * 60 * 1000);
+ }
+
+ public static void waitForZookeeperAndHdfs(VolumeManager fs) {
+ log.info("Attempting to talk to zookeeper");
+ while (true) {
+ try {
+ ZooReaderWriter.getInstance().getChildren(Constants.ZROOT);
+ break;
+ } catch (InterruptedException e) {
+ // ignored
+ } catch (KeeperException ex) {
+ log.info("Waiting for accumulo to be initialized");
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
+ long sleep = 1000;
+ int unknownHostTries = 3;
+ while (true) {
+ try {
+ if (fs.isReady())
+ break;
+ log.warn("Waiting for the NameNode to leave safemode");
+ } catch (IOException ex) {
+ log.warn("Unable to connect to HDFS", ex);
+ } catch (IllegalArgumentException exception) {
+ /* Unwrap the UnknownHostException so we can deal with it directly */
+ if (exception.getCause() instanceof UnknownHostException) {
+ if (unknownHostTries > 0) {
+ log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause());
+ /* We need to make sure our sleep period is long enough to avoid getting a cached failure of the host lookup. */
+ sleep = Math.max(sleep, (AddressUtil.getAddressCacheNegativeTtl((UnknownHostException)(exception.getCause()))+1)*1000);
+ } else {
+ log.error("Unable to connect to HDFS and have exceeded max number of retries.", exception);
+ throw exception;
+ }
+ unknownHostTries--;
+ } else {
+ throw exception;
+ }
+ }
+ log.info("Backing off due to failure; current sleep period is " + sleep / 1000. + " seconds");
+ UtilWaitThread.sleep(sleep);
+ /* Back off to give transient failures more time to clear. */
+ sleep = Math.min(60 * 1000, sleep * 2);
+ }
+ log.info("Connected to HDFS");
+ }
+
+ /**
+ * Exit loudly if there are outstanding Fate operations.
+ * Since Fate serializes class names, we need to make sure there are no queued
+ * transactions from a previous version before continuing an upgrade. The status of the operations is
+ * irrelevant; those in SUCCESSFUL status cause the same problem as those just queued.
+ *
+ * Note that the Master should not allow write access to Fate until after all upgrade steps are complete.
+ *
+ * Should be called as a guard before performing any upgrade steps, after determining that an upgrade is needed.
+ *
+ * see ACCUMULO-2519
+ */
+ public static void abortIfFateTransactions() {
+ try {
+ final ReadOnlyTStore<Accumulo> fate = new ReadOnlyStore<Accumulo>(new ZooStore<Accumulo>(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZFATE,
+ ZooReaderWriter.getInstance()));
+ if (!(fate.list().isEmpty())) {
+ throw new AccumuloException("Aborting upgrade because there are outstanding FATE transactions from a previous Accumulo version. Please see the README document for instructions on what to do under your previous version.");
+ }
+ } catch (Exception exception) {
+ log.fatal("Problem verifying Fate readiness", exception);
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 3508164,0000000..05f3c6c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -1,198 -1,0 +1,200 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.client;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.ConnectorImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An implementation of Instance that looks in HDFS and ZooKeeper to find the master and root tablet location.
+ *
+ */
+public class HdfsZooInstance implements Instance {
+
+ private HdfsZooInstance() {
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ zooCache = new ZooCacheFactory().getZooCache(acuConf.get(Property.INSTANCE_ZK_HOST), (int) acuConf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ private static HdfsZooInstance cachedHdfsZooInstance = null;
+
+ public static synchronized Instance getInstance() {
+ if (cachedHdfsZooInstance == null)
+ cachedHdfsZooInstance = new HdfsZooInstance();
+ return cachedHdfsZooInstance;
+ }
+
+ private static ZooCache zooCache;
+ private static String instanceId = null;
+ private static final Logger log = Logger.getLogger(HdfsZooInstance.class);
+
+ @Override
+ public String getRootTabletLocation() {
+ String zRootLocPath = ZooUtil.getRoot(this) + RootTable.ZROOT_TABLET_LOCATION;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zoocache.");
+
+ byte[] loc = zooCache.get(zRootLocPath);
+
- opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++ opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return null;
+ }
+
- return new String(loc, Constants.UTF8).split("\\|")[0];
++ return new String(loc, UTF_8).split("\\|")[0];
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+
+ String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
+
+ byte[] loc = ZooLock.getLockData(zooCache, masterLocPath, null);
+
- opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++ opTimer.stop("Found master at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return Collections.emptyList();
+ }
+
- return Collections.singletonList(new String(loc, Constants.UTF8));
++ return Collections.singletonList(new String(loc, UTF_8));
+ }
+
+ @Override
+ public String getInstanceID() {
+ if (instanceId == null)
+ _getInstanceID();
+ return instanceId;
+ }
+
+ private static synchronized void _getInstanceID() {
+ if (instanceId == null) {
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+ log.trace("Looking for instanceId from " + instanceIdPath);
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
+ instanceId = instanceIdFromFile;
+ }
+ }
+
+ @Override
+ public String getInstanceName() {
+ return ZooKeeperInstance.lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return (int) ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ }
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ return new ConnectorImpl(this, new Credentials(principal, token));
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, new PasswordToken(pass));
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, ByteBufferUtil.toBytes(pass));
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
+ }
+
+ private AccumuloConfiguration conf = null;
+
+ @Deprecated
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ if (conf == null)
+ conf = new ServerConfiguration(this).getConfiguration();
+ return conf;
+ }
+
+ @Override
+ @Deprecated
+ public void setConfiguration(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ public static void main(String[] args) {
+ Instance instance = HdfsZooInstance.getInstance();
+ System.out.println("Instance Name: " + instance.getInstanceName());
+ System.out.println("Instance ID: " + instance.getInstanceID());
+ System.out.println("ZooKeepers: " + instance.getZooKeepers());
+ System.out.println("Masters: " + StringUtil.join(instance.getMasterLocations(), ", "));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
index df1be50,0000000..18feb4f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
@@@ -1,160 -1,0 +1,161 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.log4j.Logger;
+
+/**
+ * A helper object for accessing properties in a {@link ZooCache}.
+ */
+public class ZooCachePropertyAccessor {
+ private static final Logger log = Logger.getLogger(ZooCachePropertyAccessor.class);
+
+ static class PropCacheKey {
+ final String instanceId;
+ final String scope;
+
+ PropCacheKey(String iid, String s) {
+ instanceId = iid;
+ scope = s;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof PropCacheKey)) {
+ return false;
+ }
+ PropCacheKey o = (PropCacheKey) other;
+ return (instanceId.equals(o.instanceId) && scope.equals(o.scope));
+ }
+
+ @Override
+ public int hashCode() {
+ int c = 17;
+ c = (37 * c) + instanceId.hashCode();
+ c = (37 * c) + scope.hashCode();
+ return c;
+ }
+ }
+
+ private final ZooCache propCache;
+
+ /**
+ * Creates a new accessor.
+ *
+ * @param propCache
+ * property cache
+ */
+ ZooCachePropertyAccessor(ZooCache propCache) {
+ this.propCache = propCache;
+ }
+
+ /**
+ * Gets the property cache accessed by this object.
+ *
+ * @return property cache
+ */
+ ZooCache getZooCache() {
+ return propCache;
+ }
+
+ /**
+ * Gets a property. If the property is not in ZooKeeper or is present but an invalid format for the property type, the parent configuration is consulted (if
+ * provided).
+ *
+ * @param property
+ * property to get
+ * @param path
+ * ZooKeeper path where properties lie
+ * @param parent
+ * parent configuration (optional)
+ * @return property value, or null if not found
+ */
+ String get(Property property, String path, AccumuloConfiguration parent) {
+ String key = property.getKey();
+ String value = get(path + "/" + key);
+
+ if (value == null || !property.getType().isValidFormat(value)) {
+ if (value != null) {
+ log.error("Using default value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
+ }
+ if (parent != null) {
+ value = parent.get(property);
+ }
+ }
+ return value;
+ }
+
+ private String get(String path) {
+ byte[] v = propCache.get(path);
+ if (v != null) {
- return new String(v, Constants.UTF8);
++ return new String(v, UTF_8);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Gets all properties into the given map. Properties are filtered using the given filter. Properties from a parent configuration are also added to the map
+ * and filtered, either using a separate filter or, if not specified, the other filter.
+ *
+ * @param props
+ * map to populate with properties
+ * @param path
+ * ZooKeeper path where properties lie
+ * @param filter
+ * property filter (required)
+ * @param parent
+ * parent configuration (required)
+ * @param parentFilter
+ * separate filter for parent properties (optional)
+ */
+ void getProperties(Map<String,String> props, String path, PropertyFilter filter, AccumuloConfiguration parent, PropertyFilter parentFilter) {
+ parent.getProperties(props, parentFilter != null ? parentFilter : filter);
+
+ List<String> children = propCache.getChildren(path);
+ if (children != null) {
+ for (String child : children) {
+ if (child != null && filter.accept(child)) {
+ String value = get(path + "/" + child);
+ if (value != null) {
+ props.put(child, value);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Clears the internal {@link ZooCache}.
+ */
+ void invalidateCache() {
+ propCache.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index a49ec43,0000000..6d25f1d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@@ -1,118 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.log4j.Logger;
+
+public class ZooConfiguration extends AccumuloConfiguration {
+ private static final Logger log = Logger.getLogger(ZooConfiguration.class);
+
+ private final String instanceId;
+ private final ZooCache propCache;
+ private final AccumuloConfiguration parent;
+ private final Map<String,String> fixedProps = Collections.synchronizedMap(new HashMap<String,String>());
+
+ protected ZooConfiguration(String instanceId, ZooCache propCache, AccumuloConfiguration parent) {
+ this.instanceId = instanceId;
+ this.propCache = propCache;
+ this.parent = parent;
+ }
+
+ @Override
+ public void invalidateCache() {
+ if (propCache != null)
+ propCache.clear();
+ }
+
+ /**
+ * Gets the parent configuration of this configuration.
+ *
+ * @return parent configuration
+ */
+ public AccumuloConfiguration getParentConfiguration() {
+ return parent;
+ }
+
+ private String _get(Property property) {
+ String key = property.getKey();
+ String value = null;
+
+ if (Property.isValidZooPropertyKey(key)) {
+ value = get(key);
+ }
+
+ if (value == null || !property.getType().isValidFormat(value)) {
+ if (value != null)
+ log.error("Using parent value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
+ value = parent.get(property);
+ }
+ return value;
+ }
+
+ @Override
+ public String get(Property property) {
+ if (Property.isFixedZooPropertyKey(property)) {
+ if (fixedProps.containsKey(property.getKey())) {
+ return fixedProps.get(property.getKey());
+ } else {
+ synchronized (fixedProps) {
+ String val = _get(property);
+ fixedProps.put(property.getKey(), val);
+ return val;
+ }
+
+ }
+ } else {
+ return _get(property);
+ }
+ }
+
+ private String get(String key) {
+ String zPath = ZooUtil.getRoot(instanceId) + Constants.ZCONFIG + "/" + key;
+ byte[] v = propCache.get(zPath);
+ String value = null;
+ if (v != null)
- value = new String(v, Constants.UTF8);
++ value = new String(v, UTF_8);
+ return value;
+ }
+
+ @Override
+ public void getProperties(Map<String,String> props, PropertyFilter filter) {
+ parent.getProperties(props, filter);
+
+ List<String> children = propCache.getChildren(ZooUtil.getRoot(instanceId) + Constants.ZCONFIG);
+ if (children != null) {
+ for (String child : children) {
+ if (child != null && filter.accept(child)) {
+ String value = get(child);
+ if (value != null)
+ props.put(child, value);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 807e821,0000000..587e38a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -1,315 -1,0 +1,317 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.constraints;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetadataConstraints implements Constraint {
+
+ private ZooCache zooCache = null;
+ private String zooRoot = null;
+
+ private static final Logger log = Logger.getLogger(MetadataConstraints.class);
+
+ private static boolean[] validTableNameChars = new boolean[256];
+
+ {
+ for (int i = 0; i < 256; i++) {
+ validTableNameChars[i] = ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) || i == '!';
+ }
+ }
+
+ private static final HashSet<ColumnFQ> validColumnQuals = new HashSet<ColumnFQ>(Arrays.asList(new ColumnFQ[] {
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN,
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN,
+ TabletsSection.ServerColumnFamily.LOCK_COLUMN, TabletsSection.ServerColumnFamily.FLUSH_COLUMN, TabletsSection.ServerColumnFamily.COMPACT_COLUMN}));
+
+ private static final HashSet<Text> validColumnFams = new HashSet<Text>(Arrays.asList(new Text[] {TabletsSection.BulkFileColumnFamily.NAME,
+ LogColumnFamily.NAME, ScanFileColumnFamily.NAME, DataFileColumnFamily.NAME,
+ TabletsSection.CurrentLocationColumnFamily.NAME, TabletsSection.LastLocationColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME,
+ ChoppedColumnFamily.NAME, ClonedColumnFamily.NAME}));
+
+ private static boolean isValidColumn(ColumnUpdate cu) {
+
+ if (validColumnFams.contains(new Text(cu.getColumnFamily())))
+ return true;
+
+ if (validColumnQuals.contains(new ColumnFQ(cu)))
+ return true;
+
+ return false;
+ }
+
+ static private ArrayList<Short> addViolation(ArrayList<Short> lst, int violation) {
+ if (lst == null)
+ lst = new ArrayList<Short>();
+ lst.add((short) violation);
+ return lst;
+ }
+
+ static private ArrayList<Short> addIfNotPresent(ArrayList<Short> lst, int intViolation) {
+ if (lst == null)
+ return addViolation(lst, intViolation);
+ short violation = (short) intViolation;
+ if (!lst.contains(violation))
+ return addViolation(lst, intViolation);
+ return lst;
+ }
+
+ @Override
+ public List<Short> check(Environment env, Mutation mutation) {
+
+ ArrayList<Short> violations = null;
+
+ Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
+
+ // check the row, it should contains at least one ; or end with <
+ boolean containsSemiC = false;
+
+ byte[] row = mutation.getRow();
+
+ // always allow rows that fall within reserved areas
+ if (row.length > 0 && row[0] == '~')
+ return null;
+ if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
+ return null;
+
+ for (byte b : row) {
+ if (b == ';') {
+ containsSemiC = true;
+ }
+
+ if (b == ';' || b == '<')
+ break;
+
+ if (!validTableNameChars[0xff & b]) {
+ violations = addIfNotPresent(violations, 4);
+ }
+ }
+
+ if (!containsSemiC) {
+ // see if last row char is <
+ if (row.length == 0 || row[row.length - 1] != '<') {
+ violations = addIfNotPresent(violations, 4);
+ }
+ } else {
+ if (row.length == 0) {
+ violations = addIfNotPresent(violations, 4);
+ }
+ }
+
+ if (row.length > 0 && row[0] == '!') {
+ if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
+ violations = addIfNotPresent(violations, 4);
+ }
+ }
+
+ // ensure row is not less than Constants.METADATA_TABLE_ID
+ if (new Text(row).compareTo(new Text(MetadataTable.ID)) < 0) {
+ violations = addViolation(violations, 5);
+ }
+
+ boolean checkedBulk = false;
+
+ for (ColumnUpdate columnUpdate : colUpdates) {
+ Text columnFamily = new Text(columnUpdate.getColumnFamily());
+
+ if (columnUpdate.isDeleted()) {
+ if (!isValidColumn(columnUpdate)) {
+ violations = addViolation(violations, 2);
+ }
+ continue;
+ }
+
+ if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
+ violations = addViolation(violations, 6);
+ }
+
+ if (columnFamily.equals(DataFileColumnFamily.NAME)) {
+ try {
+ DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
+
+ if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
+ violations = addViolation(violations, 1);
+ }
+ } catch (NumberFormatException nfe) {
+ violations = addViolation(violations, 1);
+ } catch (ArrayIndexOutOfBoundsException aiooe) {
+ violations = addViolation(violations, 1);
+ }
+ } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
+
+ } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
+ if (!columnUpdate.isDeleted() && !checkedBulk) {
+ // splits, which also write the time reference, are allowed to write this reference even when
+ // the transaction is not running because the other half of the tablet is holding a reference
+ // to the file.
+ boolean isSplitMutation = false;
+ // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
+ // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
+ // See ACCUMULO-1230.
+ boolean isLocationMutation = false;
+
+ HashSet<Text> dataFiles = new HashSet<Text>();
+ HashSet<Text> loadedFiles = new HashSet<Text>();
+
- String tidString = new String(columnUpdate.getValue(), Constants.UTF8);
++ String tidString = new String(columnUpdate.getValue(), UTF_8);
+ int otherTidCount = 0;
+
+ for (ColumnUpdate update : mutation.getUpdates()) {
+ if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
+ isSplitMutation = true;
+ } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
+ isLocationMutation = true;
+ } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
+ dataFiles.add(new Text(update.getColumnQualifier()));
+ } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
+ loadedFiles.add(new Text(update.getColumnQualifier()));
+
- if (!new String(update.getValue(), Constants.UTF8).equals(tidString)) {
++ if (!new String(update.getValue(), UTF_8).equals(tidString)) {
+ otherTidCount++;
+ }
+ }
+ }
+
+ if (!isSplitMutation && !isLocationMutation) {
+ long tid = Long.parseLong(tidString);
+
+ try {
+ if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
+ violations = addViolation(violations, 8);
+ }
+ } catch (Exception ex) {
+ violations = addViolation(violations, 8);
+ }
+ }
+
+ checkedBulk = true;
+ }
+ } else {
+ if (!isValidColumn(columnUpdate)) {
+ violations = addViolation(violations, 2);
+ } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0
+ && (violations == null || !violations.contains((short) 4))) {
+ KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
+
+ Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
+
+ boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
+
+ if (!prevEndRowLessThanEndRow) {
+ violations = addViolation(violations, 3);
+ }
+ } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
+ if (zooCache == null) {
+ zooCache = new ZooCache();
+ }
+
+ if (zooRoot == null) {
+ zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
+ }
+
+ boolean lockHeld = false;
- String lockId = new String(columnUpdate.getValue(), Constants.UTF8);
++ String lockId = new String(columnUpdate.getValue(), UTF_8);
+
+ try {
+ lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
+ } catch (Exception e) {
+ log.debug("Failed to verify lock was held " + lockId + " " + e.getMessage());
+ }
+
+ if (!lockHeld) {
+ violations = addViolation(violations, 7);
+ }
+ }
+
+ }
+ }
+
+ if (violations != null) {
- log.debug("violating metadata mutation : " + new String(mutation.getRow(), Constants.UTF8));
++ log.debug("violating metadata mutation : " + new String(mutation.getRow(), UTF_8));
+ for (ColumnUpdate update : mutation.getUpdates()) {
- log.debug(" update: " + new String(update.getColumnFamily(), Constants.UTF8) + ":" + new String(update.getColumnQualifier(), Constants.UTF8) + " value "
- + (update.isDeleted() ? "[delete]" : new String(update.getValue(), Constants.UTF8)));
++ log.debug(" update: " + new String(update.getColumnFamily(), UTF_8) + ":" + new String(update.getColumnQualifier(), UTF_8) + " value "
++ + (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
+ }
+ }
+
+ return violations;
+ }
+
+ protected Arbitrator getArbitrator() {
+ return new ZooArbitrator();
+ }
+
+ @Override
+ public String getViolationDescription(short violationCode) {
+ switch (violationCode) {
+ case 1:
+ return "data file size must be a non-negative integer";
+ case 2:
+ return "Invalid column name given.";
+ case 3:
+ return "Prev end row is greater than or equal to end row.";
+ case 4:
+ return "Invalid metadata row format";
+ case 5:
+ return "Row can not be less than " + MetadataTable.ID;
+ case 6:
+ return "Empty values are not allowed for any " + MetadataTable.NAME + " column";
+ case 7:
+ return "Lock not held in zookeeper by writer";
+ case 8:
+ return "Bulk load transaction no longer running";
+ }
+ return null;
+ }
+
+ @Override
+ protected void finalize() {
+ if (zooCache != null)
+ zooCache.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 24b5605,0000000..7c4b8d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@@ -1,586 -1,0 +1,587 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.init;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.constraints.MetadataConstraints;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * This class is used to setup the directory structure and the root tablet to get an instance started
+ *
+ */
+public class Initialize {
+ private static final Logger log = Logger.getLogger(Initialize.class);
+ private static final String DEFAULT_ROOT_USER = "root";
+ public static final String TABLE_TABLETS_TABLET_DIR = "/table_info";
+
+ private static ConsoleReader reader = null;
+ private static IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ private static ConsoleReader getConsoleReader() throws IOException {
+ if (reader == null)
+ reader = new ConsoleReader();
+ return reader;
+ }
+
+ /**
+ * Sets this class's ZooKeeper reader/writer.
+ *
+ * @param izoo
+ * reader/writer
+ */
+ static void setZooReaderWriter(IZooReaderWriter izoo) {
+ zoo = izoo;
+ }
+
+ /**
+ * Gets this class's ZooKeeper reader/writer.
+ *
+ * @return reader/writer
+ */
+ static IZooReaderWriter getZooReaderWriter() {
+ return zoo;
+ }
+
+ private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
+ static {
+ initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K");
+ initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
+ initialMetadataConf.put(Property.TABLE_WALOG_ENABLED.getKey(), "true");
+ initialMetadataConf.put(Property.TABLE_MAJC_RATIO.getKey(), "1");
+ initialMetadataConf.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "64M");
+ initialMetadataConf.put(Property.TABLE_CONSTRAINT_PREFIX.getKey() + "1", MetadataConstraints.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers", "10," + VersioningIterator.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers.opt.maxVersions", "1");
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers", "10," + VersioningIterator.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers.opt.maxVersions", "1");
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers", "10," + VersioningIterator.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers.opt.maxVersions", "1");
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.bulkLoadFilter", "20," + MetadataBulkLoadFilter.class.getName());
+ initialMetadataConf.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
+ String.format("%s,%s", TabletsSection.TabletColumnFamily.NAME, TabletsSection.CurrentLocationColumnFamily.NAME));
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s", TabletsSection.DataFileColumnFamily.NAME,
+ TabletsSection.LogColumnFamily.NAME, TabletsSection.ServerColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME));
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
+ initialMetadataConf.put(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
+ initialMetadataConf.put(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
+ initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
+ }
+
+ static boolean checkInit(Configuration conf, VolumeManager fs, SiteConfiguration sconf) throws IOException {
+ @SuppressWarnings("deprecation")
+ String fsUri = sconf.get(Property.INSTANCE_DFS_URI);
+ if (fsUri.equals(""))
+ fsUri = FileSystem.getDefaultUri(conf).toString();
+ log.info("Hadoop Filesystem is " + fsUri);
+ log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
+ log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
+ log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
+ if (!zookeeperAvailable()) {
+ log.fatal("Zookeeper needs to be up and running in order to init. Exiting ...");
+ return false;
+ }
+ if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) {
+ ConsoleReader c = getConsoleReader();
+ c.beep();
+ c.println();
+ c.println();
+ c.println("Warning!!! Your instance secret is still set to the default, this is not secure. We highly recommend you change it.");
+ c.println();
+ c.println();
+ c.println("You can change the instance secret in accumulo by using:");
+ c.println(" bin/accumulo " + org.apache.accumulo.server.util.ChangeSecret.class.getName() + " oldPassword newPassword.");
+ c.println("You will also need to edit your secret in your configuration file by adding the property instance.secret to your conf/accumulo-site.xml. Without this accumulo will not operate correctly");
+ }
+ try {
+ if (isInitialized(fs)) {
+ printInitializeFailureMessages(sconf);
+ return false;
+ }
+ } catch (IOException e) {
+ throw new IOException("Failed to check if filesystem already initialized", e);
+ }
+
+ return true;
+ }
+
+ static void printInitializeFailureMessages(SiteConfiguration sconf) {
+ @SuppressWarnings("deprecation")
+ Property INSTANCE_DFS_DIR = Property.INSTANCE_DFS_DIR;
+ @SuppressWarnings("deprecation")
+ Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI;
+ String instanceDfsDir = sconf.get(INSTANCE_DFS_DIR);
+ log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()))
+ + " were previously initialized.");
+ String instanceVolumes = sconf.get(Property.INSTANCE_VOLUMES);
+ String instanceDfsUri = sconf.get(INSTANCE_DFS_URI);
+
+ if (!instanceVolumes.isEmpty()) {
+ log.fatal("Change the property " + Property.INSTANCE_VOLUMES + " to use different filesystems,");
+ } else if (!instanceDfsDir.isEmpty()) {
+ log.fatal("Change the property " + INSTANCE_DFS_URI + " to use a different filesystem,");
+ } else {
+ log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_VOLUMES + " to use a different filesystem,");
+ }
+ log.fatal("or change the property " + INSTANCE_DFS_DIR + " to use a different directory.");
+ log.fatal("The current value of " + INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|");
+ log.fatal("The current value of " + INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|");
+ log.fatal("The current value of " + Property.INSTANCE_VOLUMES + " is |" + instanceVolumes + "|");
+ }
+
+ public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException {
+ if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) {
+ return false;
+ }
+
+ // prompt user for instance name and root password early, in case they
+ // abort, we don't leave an inconsistent HDFS/ZooKeeper structure
+ String instanceNamePath;
+ try {
+ instanceNamePath = getInstanceNamePath(opts);
+ } catch (Exception e) {
+ log.fatal("Failed to talk to zookeeper", e);
+ return false;
+ }
+ opts.rootpass = getRootPassword(opts);
+ return initialize(opts, instanceNamePath, fs);
+ }
+
+ public static boolean initialize(Opts opts, String instanceNamePath, VolumeManager fs) {
+
+ UUID uuid = UUID.randomUUID();
+ // the actual disk locations of the root table and tablets
+ String[] configuredVolumes = VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration());
+ final String rootTabletDir = new Path(fs.choose(configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + RootTable.ID
+ + RootTable.ROOT_TABLET_LOCATION).toString();
+
+ try {
+ initZooKeeper(opts, uuid.toString(), instanceNamePath, rootTabletDir);
+ } catch (Exception e) {
+ log.fatal("Failed to initialize zookeeper", e);
+ return false;
+ }
+
+ try {
+ initFileSystem(opts, fs, uuid, rootTabletDir);
+ } catch (Exception e) {
+ log.fatal("Failed to initialize filesystem", e);
+
+ if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES).trim().equals("")) {
+ Configuration fsConf = CachedConfiguration.getInstance();
+
+ final String defaultFsUri = "file:///";
+ String fsDefaultName = fsConf.get("fs.default.name", defaultFsUri), fsDefaultFS = fsConf.get("fs.defaultFS", defaultFsUri);
+
+ // Try to determine when we couldn't find an appropriate core-site.xml on the classpath
+ if (defaultFsUri.equals(fsDefaultName) && defaultFsUri.equals(fsDefaultFS)) {
+ log.fatal("Default filesystem value ('fs.defaultFS' or 'fs.default.name') of '" + defaultFsUri + "' was found in the Hadoop configuration");
+ log.fatal("Please ensure that the Hadoop core-site.xml is on the classpath using 'general.classpaths' in accumulo-site.xml");
+ }
+ }
+
+ return false;
+ }
+
+ try {
+ initSecurity(opts, uuid.toString());
+ } catch (Exception e) {
+ log.fatal("Failed to initialize security", e);
+ return false;
+ }
+ return true;
+ }
+
+ private static boolean zookeeperAvailable() {
+ try {
+ return zoo.exists("/");
+ } catch (KeeperException e) {
+ return false;
+ } catch (InterruptedException e) {
+ return false;
+ }
+ }
+
+ private static void initDirs(VolumeManager fs, UUID uuid, String[] baseDirs, boolean print) throws IOException {
+ for (String baseDir : baseDirs) {
+ fs.mkdirs(new Path(new Path(baseDir, ServerConstants.VERSION_DIR), "" + ServerConstants.DATA_VERSION));
+
+ // create an instance id
+ Path iidLocation = new Path(baseDir, ServerConstants.INSTANCE_ID_DIR);
+ fs.mkdirs(iidLocation);
+ fs.createNewFile(new Path(iidLocation, uuid.toString()));
+ if (print)
+ log.info("Initialized volume " + baseDir);
+ }
+ }
+
+ private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, String rootTabletDir) throws IOException {
+ initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
+
+ // initialize initial metadata config in zookeeper
+ initMetadataConfig();
+
+ String tableMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
+ + TABLE_TABLETS_TABLET_DIR;
+ String defaultMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
+ + Constants.DEFAULT_TABLET_LOCATION;
+
+ // create table and default tablets directories
+ createDirectories(fs, rootTabletDir, tableMetadataTabletDir, defaultMetadataTabletDir);
+
+ // populate the root tablet with info about the metadata tablets
+ String fileName = rootTabletDir + Path.SEPARATOR + "00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
+ createMetadataFile(fs, fileName, MetadataTable.ID, tableMetadataTabletDir, defaultMetadataTabletDir);
+ }
+
+ /**
+ * Create an rfile in the default tablet's directory for a new table. This method is used to create the initial root tablet contents, with information about
+ * the metadata table's tablets
+ *
+ * @param volmanager
+ * The VolumeManager
+ * @param fileName
+ * The location to create the file
+ * @param tableId
+ * TableID that is being "created"
+ * @param tableTabletDir
+ * The table_info directory for the new table
+ * @param defaultTabletDir
+ * The default_tablet directory for the new table
+ */
+ private static void createMetadataFile(VolumeManager volmanager, String fileName, String tableId, String tableTabletDir, String defaultTabletDir)
+ throws IOException {
+ FileSystem fs = volmanager.getVolumeByPath(new Path(fileName)).getFileSystem();
+ FileSKVWriter tabletWriter = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
+ tabletWriter.startDefaultLocalityGroup();
+
+ Text splitPoint = TabletsSection.getRange().getEndKey().getRow();
+ createEntriesForTablet(tabletWriter, tableId, tableTabletDir, null, splitPoint);
+ createEntriesForTablet(tabletWriter, tableId, defaultTabletDir, splitPoint, null);
+
+ tabletWriter.close();
+ }
+
+ private static void createEntriesForTablet(FileSKVWriter writer, String tableId, String tabletDir, Text tabletPrevEndRow, Text tabletEndRow)
+ throws IOException {
+ Text extent = new Text(KeyExtent.getMetadataEntry(new Text(tableId), tabletEndRow));
- addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(Constants.UTF8)));
- addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(Constants.UTF8)));
++ addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(UTF_8)));
++ addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8)));
+ addEntry(writer, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tabletPrevEndRow));
+ }
+
+ private static void addEntry(FileSKVWriter writer, Text row, ColumnFQ col, Value value) throws IOException {
+ writer.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier(), 0), value);
+ }
+
+ private static void createDirectories(VolumeManager fs, String... dirs) throws IOException {
+ for (String s : dirs) {
+ Path dir = new Path(s);
+ try {
+ FileStatus fstat = fs.getFileStatus(dir);
+ // TODO Remove deprecation warning suppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ boolean isDirectory = fstat.isDir();
+ if (!isDirectory) {
+ log.fatal("location " + dir + " exists but is not a directory");
+ return;
+ }
+ } catch (FileNotFoundException fnfe) {
+ // attempt to create directory, since it doesn't exist
+ if (!fs.mkdirs(dir)) {
+ log.fatal("unable to create directory " + dir);
+ return;
+ }
+ }
+ }
+ }
+
+ private static void initZooKeeper(Opts opts, String uuid, String instanceNamePath, String rootTabletDir) throws KeeperException, InterruptedException {
+ // setup basic data in zookeeper
+ zoo.putPersistentData(Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
+ zoo.putPersistentData(Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
+
+ // setup instance name
+ if (opts.clearInstanceName)
+ zoo.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
- zoo.putPersistentData(instanceNamePath, uuid.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(instanceNamePath, uuid.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ final byte[] EMPTY_BYTE_ARRAY = new byte[0], ZERO_CHAR_ARRAY = new byte[] {'0'};
+
+ // setup the instance
+ String zkInstanceRoot = Constants.ZROOT + "/" + uuid;
+ zoo.putPersistentData(zkInstanceRoot, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES, Constants.ZTABLES_INITIAL_ID, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES, new byte[0], NodeExistsPolicy.FAIL);
+ TableManager.prepareNewNamespaceState(uuid, Namespaces.DEFAULT_NAMESPACE_ID, Namespaces.DEFAULT_NAMESPACE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewNamespaceState(uuid, Namespaces.ACCUMULO_NAMESPACE_ID, Namespaces.ACCUMULO_NAMESPACE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, MetadataTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, MetadataTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTSERVERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZPROBLEMS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_WALOGS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8), NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZNEXT_FILE, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZRECOVERY, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ }
+
+ private static String getInstanceNamePath(Opts opts) throws IOException, KeeperException, InterruptedException {
+ // setup the instance name
+ String instanceName, instanceNamePath = null;
+ boolean exists = true;
+ do {
+ if (opts.cliInstanceName == null) {
+ instanceName = getConsoleReader().readLine("Instance name : ");
+ } else {
+ instanceName = opts.cliInstanceName;
+ }
+ if (instanceName == null)
+ System.exit(0);
+ instanceName = instanceName.trim();
+ if (instanceName.length() == 0)
+ continue;
+ instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
+ if (opts.clearInstanceName) {
+ exists = false;
+ break;
+ } else if (exists = zoo.exists(instanceNamePath)) {
+ String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : ");
+ if (decision == null)
+ System.exit(0);
+ if (decision.length() == 1 && decision.toLowerCase(Locale.ENGLISH).charAt(0) == 'y') {
+ opts.clearInstanceName = true;
+ exists = false;
+ }
+ }
+ } while (exists);
+ return instanceNamePath;
+ }
+
+ private static byte[] getRootPassword(Opts opts) throws IOException {
+ if (opts.cliPassword != null) {
- return opts.cliPassword.getBytes(Constants.UTF8);
++ return opts.cliPassword.getBytes(UTF_8);
+ }
+ String rootpass;
+ String confirmpass;
+ do {
+ rootpass = getConsoleReader()
+ .readLine("Enter initial password for " + DEFAULT_ROOT_USER + " (this may not be applicable for your security setup): ", '*');
+ if (rootpass == null)
+ System.exit(0);
+ confirmpass = getConsoleReader().readLine("Confirm initial password for " + DEFAULT_ROOT_USER + ": ", '*');
+ if (confirmpass == null)
+ System.exit(0);
+ if (!rootpass.equals(confirmpass))
+ log.error("Passwords do not match");
+ } while (!rootpass.equals(confirmpass));
- return rootpass.getBytes(Constants.UTF8);
++ return rootpass.getBytes(UTF_8);
+ }
+
+ private static void initSecurity(Opts opts, String iid) throws AccumuloSecurityException, ThriftSecurityException {
+ AuditedSecurityOperation.getInstance(iid, true).initializeSecurity(SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()), DEFAULT_ROOT_USER,
+ opts.rootpass);
+ }
+
+ public static void initMetadataConfig() throws IOException {
+ try {
+ Configuration conf = CachedConfiguration.getInstance();
+ int max = conf.getInt("dfs.replication.max", 512);
+ // Hadoop 0.23 switched the min value configuration name
+ int min = Math.max(conf.getInt("dfs.replication.min", 1), conf.getInt("dfs.namenode.replication.min", 1));
+ if (max < 5)
+ setMetadataReplication(max, "max");
+ if (min > 5)
+ setMetadataReplication(min, "min");
+ for (Entry<String,String> entry : initialMetadataConf.entrySet()) {
+ if (!TablePropUtil.setTableProperty(RootTable.ID, entry.getKey(), entry.getValue()))
+ throw new IOException("Cannot create per-table property " + entry.getKey());
+ if (!TablePropUtil.setTableProperty(MetadataTable.ID, entry.getKey(), entry.getValue()))
+ throw new IOException("Cannot create per-table property " + entry.getKey());
+ }
+ } catch (Exception e) {
+ log.fatal("error talking to zookeeper", e);
+ throw new IOException(e);
+ }
+ }
+
+ private static void setMetadataReplication(int replication, String reason) throws IOException {
+ String rep = getConsoleReader().readLine(
+ "Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME + " replication of 5. What do you want to set your "
+ + MetadataTable.NAME + " replication to? (" + replication + ") ");
+ if (rep == null || rep.length() == 0)
+ rep = Integer.toString(replication);
+ else
+ // Lets make sure it's a number
+ Integer.parseInt(rep);
+ initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), rep);
+ }
+
+ public static boolean isInitialized(VolumeManager fs) throws IOException {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
+ if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR)))
+ return true;
+ }
+
+ return false;
+ }
+
+ private static void addVolumes(VolumeManager fs) throws IOException {
+ HashSet<String> initializedDirs = new HashSet<String>();
+ initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
+
+ HashSet<String> uinitializedDirs = new HashSet<String>();
+ uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
+ uinitializedDirs.removeAll(initializedDirs);
+
+ Path aBasePath = new Path(initializedDirs.iterator().next());
+ Path iidPath = new Path(aBasePath, ServerConstants.INSTANCE_ID_DIR);
+ Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
+
+ UUID uuid = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(iidPath, ServerConfiguration.getSiteConfiguration()));
+
+ if (ServerConstants.DATA_VERSION != Accumulo.getAccumuloPersistentVersion(versionPath.getFileSystem(CachedConfiguration.getInstance()), versionPath)) {
+ throw new IOException("Accumulo " + Constants.VERSION + " cannot initialize data version " + Accumulo.getAccumuloPersistentVersion(fs));
+ }
+
+ initDirs(fs, uuid, uinitializedDirs.toArray(new String[uinitializedDirs.size()]), true);
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = "--add-volumes", description = "Initialize any uninitialized volumes listed in instance.volumes")
+ boolean addVolumes = false;
+ @Parameter(names = "--reset-security", description = "just update the security information")
+ boolean resetSecurity = false;
+ @Parameter(names = "--clear-instance-name", description = "delete any existing instance name without prompting")
+ boolean clearInstanceName = false;
+ @Parameter(names = "--instance-name", description = "the instance name, if not provided, will prompt")
+ String cliInstanceName;
+ @Parameter(names = "--password", description = "set the password on the command line")
+ String cliPassword;
+
+ byte[] rootpass = null;
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(Initialize.class.getName(), args);
+
+ try {
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ SecurityUtil.serverLogin(acuConf);
+ Configuration conf = CachedConfiguration.getInstance();
+
+ VolumeManager fs = VolumeManagerImpl.get(acuConf);
+
+ if (opts.resetSecurity) {
+ if (isInitialized(fs)) {
+ opts.rootpass = getRootPassword(opts);
+ initSecurity(opts, HdfsZooInstance.getInstance().getInstanceID());
+ } else {
+ log.fatal("Attempted to reset security on accumulo before it was initialized");
+ }
+ }
+
+ if (opts.addVolumes) {
+ addVolumes(fs);
+ }
+
+ if (!opts.resetSecurity && !opts.addVolumes)
+ if (!doInit(opts, conf, fs))
+ System.exit(-1);
+ } catch (Exception e) {
+ log.fatal(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+}
[14/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
index d0e6aea,0000000..258080c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
@@@ -1,775 -1,0 +1,777 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.SecurityOperationsImpl;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.master.thrift.FateOperation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.server.security.handler.Authorizor;
+import org.apache.accumulo.server.security.handler.PermissionHandler;
+import org.apache.accumulo.server.security.handler.ZKAuthenticator;
+import org.apache.accumulo.server.security.handler.ZKAuthorizor;
+import org.apache.accumulo.server.security.handler.ZKPermHandler;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * Utility class for performing various security operations with the appropriate checks
+ */
+public class SecurityOperation {
+ private static final Logger log = Logger.getLogger(SecurityOperationsImpl.class);
+
+ protected Authorizor authorizor;
+ protected Authenticator authenticator;
+ protected PermissionHandler permHandle;
+ private static String rootUserName = null;
+ private final ZooCache zooCache;
+ private final String ZKUserPath;
+
+ static SecurityOperation instance;
+
+ public static synchronized SecurityOperation getInstance() {
+ String instanceId = HdfsZooInstance.getInstance().getInstanceID();
+ return getInstance(instanceId, false);
+ }
+
+ public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) {
+ if (instance == null) {
+ instance = new SecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId, initialize),
+ instanceId);
+ }
+ return instance;
+ }
+
+ protected static Authorizor getAuthorizor(String instanceId, boolean initialize) {
+ Authorizor toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHORIZOR, Authorizor.class,
+ ZKAuthorizor.getInstance());
+ toRet.initialize(instanceId, initialize);
+ return toRet;
+ }
+
+ protected static Authenticator getAuthenticator(String instanceId, boolean initialize) {
+ Authenticator toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_AUTHENTICATOR, Authenticator.class,
+ ZKAuthenticator.getInstance());
+ toRet.initialize(instanceId, initialize);
+ return toRet;
+ }
+
+ protected static PermissionHandler getPermHandler(String instanceId, boolean initialize) {
+ PermissionHandler toRet = ServerConfiguration.getSiteConfiguration().instantiateClassProperty(Property.INSTANCE_SECURITY_PERMISSION_HANDLER,
+ PermissionHandler.class, ZKPermHandler.getInstance());
+ toRet.initialize(instanceId, initialize);
+ return toRet;
+ }
+
+ protected SecurityOperation(String instanceId) {
+ ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
+ zooCache = new ZooCache();
+ }
+
+ public SecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) {
+ this(instanceId);
+ authorizor = author;
+ authenticator = authent;
+ permHandle = pm;
+
+ if (!authorizor.validSecurityHandlers(authenticator, pm) || !authenticator.validSecurityHandlers(authorizor, pm)
+ || !permHandle.validSecurityHandlers(authent, author))
+ throw new RuntimeException(authorizor + ", " + authenticator + ", and " + pm
+ + " do not play nice with eachother. Please choose authentication and authorization mechanisms that are compatible with one another.");
+ }
+
+ public void initializeSecurity(TCredentials credentials, String rootPrincipal, byte[] token) throws AccumuloSecurityException, ThriftSecurityException {
+ authenticate(credentials);
+
+ if (!isSystemUser(credentials))
+ throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ authenticator.initializeSecurity(credentials, rootPrincipal, token);
+ authorizor.initializeSecurity(credentials, rootPrincipal);
+ permHandle.initializeSecurity(credentials, rootPrincipal);
+ try {
+ permHandle.grantTablePermission(rootPrincipal, MetadataTable.ID, TablePermission.ALTER_TABLE);
+ } catch (TableNotFoundException e) {
+ // Shouldn't happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ public synchronized String getRootUsername() {
+ if (rootUserName == null)
- rootUserName = new String(zooCache.get(ZKUserPath), Constants.UTF8);
++ rootUserName = new String(zooCache.get(ZKUserPath), UTF_8);
+ return rootUserName;
+ }
+
+ public boolean isSystemUser(TCredentials credentials) {
+ return SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName());
+ }
+
+ protected void authenticate(TCredentials credentials) throws ThriftSecurityException {
+ if (!credentials.getInstanceId().equals(HdfsZooInstance.getInstance().getInstanceID()))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.INVALID_INSTANCEID);
+
+ Credentials creds = Credentials.fromThrift(credentials);
+ if (isSystemUser(credentials)) {
+ if (!(SystemCredentials.get().equals(creds))) {
+ throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
+ }
+ } else {
+ try {
+ if (!authenticator.authenticateUser(creds.getPrincipal(), creds.getToken())) {
+ throw new ThriftSecurityException(creds.getPrincipal(), SecurityErrorCode.BAD_CREDENTIALS);
+ }
+ } catch (AccumuloSecurityException e) {
+ log.debug(e);
+ throw e.asThriftException();
+ }
+ }
+ }
+
+ public boolean canAskAboutUser(TCredentials credentials, String user) throws ThriftSecurityException {
+ // Authentication done in canPerformSystemActions
+ if (!(canPerformSystemActions(credentials) || credentials.getPrincipal().equals(user)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return true;
+ }
+
+ public boolean authenticateUser(TCredentials credentials, TCredentials toAuth) throws ThriftSecurityException {
+ canAskAboutUser(credentials, toAuth.getPrincipal());
+ // User is already authenticated from canAskAboutUser
+ if (credentials.equals(toAuth))
+ return true;
+ try {
+ Credentials toCreds = Credentials.fromThrift(toAuth);
+ return authenticator.authenticateUser(toCreds.getPrincipal(), toCreds.getToken());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public Authorizations getUserAuthorizations(TCredentials credentials, String user) throws ThriftSecurityException {
+ authenticate(credentials);
+
+ targetUserExists(user);
+
+ if (!credentials.getPrincipal().equals(user) && !hasSystemPermission(credentials, SystemPermission.SYSTEM, false))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ try {
+ return authorizor.getCachedUserAuthorizations(user);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public Authorizations getUserAuthorizations(TCredentials credentials) throws ThriftSecurityException {
+ // system user doesn't need record-level authorizations for the tables it reads
+ if (isSystemUser(credentials)) {
+ authenticate(credentials);
+ return Authorizations.EMPTY;
+ }
+ return getUserAuthorizations(credentials, credentials.getPrincipal());
+ }
+
+ public boolean userHasAuthorizations(TCredentials credentials, List<ByteBuffer> list) throws ThriftSecurityException {
+ authenticate(credentials);
+
+ if (isSystemUser(credentials)) {
+ // system user doesn't need record-level authorizations for the tables it reads (for now)
+ return list.isEmpty();
+ }
+
+ try {
+ return authorizor.isValidAuthorizations(credentials.getPrincipal(), list);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ private boolean hasSystemPermission(TCredentials credentials, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
+ return hasSystemPermissionWithNamespaceId(credentials, permission, null, useCached);
+ }
+
+ /**
+ * Checks if a user has a system permission
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ private boolean hasSystemPermissionWithNamespaceId(TCredentials credentials, SystemPermission permission, String namespaceId, boolean useCached)
+ throws ThriftSecurityException {
+ if (isSystemUser(credentials))
+ return true;
+
+ if (_hasSystemPermission(credentials.getPrincipal(), permission, useCached))
+ return true;
+ if (namespaceId != null) {
+ return _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
+ }
+
+ return false;
+ }
+
+ /**
+ * Checks if a user has a system permission<br/>
+ * This cannot check if a system user has permission.
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ private boolean _hasSystemPermission(String user, SystemPermission permission, boolean useCached) throws ThriftSecurityException {
+ if (user.equals(getRootUsername()))
+ return true;
+
+ targetUserExists(user);
+
+ try {
+ if (useCached)
+ return permHandle.hasCachedSystemPermission(user, permission);
+ return permHandle.hasSystemPermission(user, permission);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ /**
+ * Checks if a user has a table permission
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ protected boolean hasTablePermission(TCredentials credentials, String tableId, String namespaceId, TablePermission permission, boolean useCached) throws ThriftSecurityException {
+ if (isSystemUser(credentials))
+ return true;
+ return _hasTablePermission(credentials.getPrincipal(), tableId, permission, useCached)
+ || _hasNamespacePermission(credentials.getPrincipal(), namespaceId, NamespacePermission.getEquivalent(permission), useCached);
+ }
+
+ /**
+ * Checks if a user has a table permission<br/>
+ * This cannot check if a system user has permission.
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ protected boolean _hasTablePermission(String user, String table, TablePermission permission, boolean useCached) throws ThriftSecurityException {
+ targetUserExists(user);
+
+ if ((table.equals(MetadataTable.ID) || table.equals(RootTable.ID)) && permission.equals(TablePermission.READ))
+ return true;
+
+ try {
+ if (useCached)
+ return permHandle.hasCachedTablePermission(user, table, permission);
+ return permHandle.hasTablePermission(user, table, permission);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(user, SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ /**
+ * Checks if a user has a namespace permission<br/>
+ * This cannot check if a system user has permission.
+ *
+ * @return true if a user exists and has permission; false otherwise
+ */
+ protected boolean _hasNamespacePermission(String user, String namespace, NamespacePermission permission, boolean useCached) throws ThriftSecurityException {
+ if (permission == null)
+ return false;
+
+ targetUserExists(user);
+
+ if (namespace.equals(Namespaces.ACCUMULO_NAMESPACE_ID) && permission.equals(NamespacePermission.READ))
+ return true;
+
+ try {
+ if (useCached)
+ return permHandle.hasCachedNamespacePermission(user, namespace, permission);
+ return permHandle.hasNamespacePermission(user, namespace, permission);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(user, SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ // some people just aren't allowed to ask about other users; here are those who can ask
+ private boolean canAskAboutOtherUsers(TCredentials credentials, String user) throws ThriftSecurityException {
+ authenticate(credentials);
+ return credentials.getPrincipal().equals(user) || hasSystemPermission(credentials, SystemPermission.SYSTEM, false)
+ || hasSystemPermission(credentials, SystemPermission.CREATE_USER, false) || hasSystemPermission(credentials, SystemPermission.ALTER_USER, false)
+ || hasSystemPermission(credentials, SystemPermission.DROP_USER, false);
+ }
+
+ private void targetUserExists(String user) throws ThriftSecurityException {
+ if (user.equals(getRootUsername()))
+ return;
+ try {
+ if (!authenticator.userExists(user))
+ throw new ThriftSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST);
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public boolean canScan(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, true);
+ }
+
+ public boolean canScan(TCredentials credentials, String tableId, String namespaceId, TRange range, List<TColumn> columns, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
+ return canScan(credentials, tableId, namespaceId);
+ }
+
+ public boolean canScan(TCredentials credentials, String table, String namespaceId, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
+ return canScan(credentials, table, namespaceId);
+ }
+
+ public boolean canWrite(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasTablePermission(credentials, tableId, namespaceId, TablePermission.WRITE, true);
+ }
+
+ public boolean canConditionallyUpdate(TCredentials credentials, String tableID, String namespaceId, List<ByteBuffer> authorizations) throws ThriftSecurityException {
+
+ authenticate(credentials);
+
+ return hasTablePermission(credentials, tableID, namespaceId, TablePermission.WRITE, true) && hasTablePermission(credentials, tableID, namespaceId, TablePermission.READ, true);
+ }
+
+ public boolean canSplitTablet(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasSystemPermissionWithNamespaceId(credentials, SystemPermission.SYSTEM, namespaceId, false)
+ || hasTablePermission(credentials, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ /**
+ * This is the check to perform any system action. This includes tserver's loading of a tablet, shutting the system down, or altering system properties.
+ */
+ public boolean canPerformSystemActions(TCredentials credentials) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermission(credentials, SystemPermission.SYSTEM, false);
+ }
+
+ public boolean canFlush(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canAlterTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false)
+ || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false);
+ }
+
+ public boolean canCreateTable(TCredentials c, String table, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, namespaceId, false);
+ }
+
+ public boolean canRenameTable(TCredentials c, String tableId, String oldTableName, String newTableName, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canCloneTable(TCredentials c, String tableId, String tableName, String destinationNamespaceId, String srcNamespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.CREATE_TABLE, destinationNamespaceId, false) && hasTablePermission(c, tableId, srcNamespaceId, TablePermission.READ, false);
+ }
+
+ public boolean canDeleteTable(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.DROP_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.DROP_TABLE, false);
+ }
+
+ public boolean canOnlineOfflineTable(TCredentials c, String tableId, FateOperation op, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
+ || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canMerge(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false)
+ || hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false);
+ }
+
+ public boolean canDeleteRange(TCredentials c, String tableId, String tableName, Text startRow, Text endRow, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.SYSTEM, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
+ }
+
+ public boolean canBulkImport(TCredentials c, String tableId, String tableName, String dir, String failDir, String namespaceId) throws ThriftSecurityException {
+ return canBulkImport(c, tableId, namespaceId);
+ }
+
+ public boolean canBulkImport(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasTablePermission(c, tableId, namespaceId, TablePermission.BULK_IMPORT, false);
+ }
+
+ public boolean canCompact(TCredentials c, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false)
+ || hasTablePermission(c, tableId, namespaceId, TablePermission.ALTER_TABLE, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.WRITE, false);
+ }
+
+ public boolean canChangeAuthorizations(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermission(c, SystemPermission.ALTER_USER, false);
+ }
+
+ public boolean canChangePassword(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ return c.getPrincipal().equals(user) || hasSystemPermission(c, SystemPermission.ALTER_USER, false);
+ }
+
+ public boolean canCreateUser(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermission(c, SystemPermission.CREATE_USER, false);
+ }
+
+ public boolean canDropUser(TCredentials c, String user) throws ThriftSecurityException {
+ authenticate(c);
+ if (user.equals(getRootUsername()))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return hasSystemPermission(c, SystemPermission.DROP_USER, false);
+ }
+
+ public boolean canGrantSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
+ authenticate(c);
+ // can't grant GRANT
+ if (sysPerm.equals(SystemPermission.GRANT))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
+ return hasSystemPermission(c, SystemPermission.GRANT, false);
+ }
+
+ public boolean canGrantTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
+ }
+
+ public boolean canGrantNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
+ return canModifyNamespacePermission(c, user, namespace);
+ }
+
+ private boolean canModifyNamespacePermission(TCredentials c, String user, String namespace) throws ThriftSecurityException {
+ authenticate(c);
+ // The one case where Table/SystemPermission -> NamespacePermission breaks down. The alternative is to make SystemPermission.ALTER_NAMESPACE provide
+ // NamespacePermission.GRANT & ALTER_NAMESPACE, but then it would cause some permission checks to succeed with GRANT when they shouldn't
+
+ // This is a bit hackier then I (vines) wanted, but I think this one hackiness makes the overall SecurityOperations more succinct.
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_NAMESPACE, namespace, false)
+ || hasNamespacePermission(c, c.principal, namespace, NamespacePermission.GRANT);
+ }
+
+ public boolean canRevokeSystem(TCredentials c, String user, SystemPermission sysPerm) throws ThriftSecurityException {
+ authenticate(c);
+ // can't modify root user
+ if (user.equals(getRootUsername()))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ // can't revoke GRANT
+ if (sysPerm.equals(SystemPermission.GRANT))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.GRANT_INVALID);
+
+ return hasSystemPermission(c, SystemPermission.GRANT, false);
+ }
+
+ public boolean canRevokeTable(TCredentials c, String user, String tableId, String namespaceId) throws ThriftSecurityException {
+ authenticate(c);
+ return hasSystemPermissionWithNamespaceId(c, SystemPermission.ALTER_TABLE, namespaceId, false) || hasTablePermission(c, tableId, namespaceId, TablePermission.GRANT, false);
+ }
+
+ public boolean canRevokeNamespace(TCredentials c, String user, String namespace) throws ThriftSecurityException {
+ return canModifyNamespacePermission(c, user, namespace);
+ }
+
+ public void changeAuthorizations(TCredentials credentials, String user, Authorizations authorizations) throws ThriftSecurityException {
+ if (!canChangeAuthorizations(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ authorizor.changeAuthorizations(user, authorizations);
+ log.info("Changed authorizations for user " + user + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException ase) {
+ throw ase.asThriftException();
+ }
+ }
+
+ public void changePassword(TCredentials credentials, Credentials toChange) throws ThriftSecurityException {
+ if (!canChangePassword(credentials, toChange.getPrincipal()))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ AuthenticationToken token = toChange.getToken();
+ authenticator.changePassword(toChange.getPrincipal(), token);
+ log.info("Changed password for user " + toChange.getPrincipal() + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void createUser(TCredentials credentials, Credentials newUser, Authorizations authorizations) throws ThriftSecurityException {
+ if (!canCreateUser(credentials, newUser.getPrincipal()))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ AuthenticationToken token = newUser.getToken();
+ authenticator.createUser(newUser.getPrincipal(), token);
+ authorizor.initUser(newUser.getPrincipal());
+ permHandle.initUser(newUser.getPrincipal());
+ log.info("Created user " + newUser.getPrincipal() + " at the request of user " + credentials.getPrincipal());
+ if (canChangeAuthorizations(credentials, newUser.getPrincipal()))
+ authorizor.changeAuthorizations(newUser.getPrincipal(), authorizations);
+ } catch (AccumuloSecurityException ase) {
+ throw ase.asThriftException();
+ }
+ }
+
+ public void dropUser(TCredentials credentials, String user) throws ThriftSecurityException {
+ if (!canDropUser(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ authorizor.dropUser(user);
+ authenticator.dropUser(user);
+ permHandle.cleanUser(user);
+ log.info("Deleted user " + user + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void grantSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
+ if (!canGrantSystem(credentials, user, permissionById))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.grantSystemPermission(user, permissionById);
+ log.info("Granted system permission " + permissionById + " for user " + user + " at the request of user " + credentials.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void grantTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
+ if (!canGrantTable(c, user, tableId, namespaceId))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.grantTablePermission(user, tableId, permission);
+ log.info("Granted table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ public void grantNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
+ if (!canGrantNamespace(c, user, namespace))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.grantNamespacePermission(user, namespace, permission);
+ log.info("Granted namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
+ + c.getPrincipal());
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ public void revokeSystemPermission(TCredentials credentials, String user, SystemPermission permission) throws ThriftSecurityException {
+ if (!canRevokeSystem(credentials, user, permission))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.revokeSystemPermission(user, permission);
+ log.info("Revoked system permission " + permission + " for user " + user + " at the request of user " + credentials.getPrincipal());
+
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void revokeTablePermission(TCredentials c, String user, String tableId, TablePermission permission, String namespaceId) throws ThriftSecurityException {
+ if (!canRevokeTable(c, user, tableId, namespaceId))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.revokeTablePermission(user, tableId, permission);
+ log.info("Revoked table permission " + permission + " for user " + user + " on the table " + tableId + " at the request of user " + c.getPrincipal());
+
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ public void revokeNamespacePermission(TCredentials c, String user, String namespace, NamespacePermission permission) throws ThriftSecurityException {
+ if (!canRevokeNamespace(c, user, namespace))
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ targetUserExists(user);
+
+ try {
+ permHandle.revokeNamespacePermission(user, namespace, permission);
+ log.info("Revoked namespace permission " + permission + " for user " + user + " on the namespace " + namespace + " at the request of user "
+ + c.getPrincipal());
+
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ public boolean hasSystemPermission(TCredentials credentials, String user, SystemPermission permissionById) throws ThriftSecurityException {
+ if (!canAskAboutOtherUsers(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return _hasSystemPermission(user, permissionById, false);
+ }
+
+ public boolean hasTablePermission(TCredentials credentials, String user, String tableId, TablePermission permissionById) throws ThriftSecurityException {
+ if (!canAskAboutOtherUsers(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return _hasTablePermission(user, tableId, permissionById, false);
+ }
+
+ public boolean hasNamespacePermission(TCredentials credentials, String user, String namespace, NamespacePermission permissionById)
+ throws ThriftSecurityException {
+ if (!canAskAboutOtherUsers(credentials, user))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ return _hasNamespacePermission(user, namespace, permissionById, false);
+ }
+
+ public Set<String> listUsers(TCredentials credentials) throws ThriftSecurityException {
+ authenticate(credentials);
+ try {
+ return authenticator.listUsers();
+ } catch (AccumuloSecurityException e) {
+ throw e.asThriftException();
+ }
+ }
+
+ public void deleteTable(TCredentials credentials, String tableId, String namespaceId) throws ThriftSecurityException {
+ if (!canDeleteTable(credentials, tableId, namespaceId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ permHandle.cleanTablePermissions(tableId);
+ } catch (AccumuloSecurityException e) {
+ e.setUser(credentials.getPrincipal());
+ throw e.asThriftException();
+ } catch (TableNotFoundException e) {
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.TABLE_DOESNT_EXIST);
+ }
+ }
+
+ public void deleteNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
+ if (!canDeleteNamespace(credentials, namespace))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ try {
+ permHandle.cleanNamespacePermissions(namespace);
+ } catch (AccumuloSecurityException e) {
+ e.setUser(credentials.getPrincipal());
+ throw e.asThriftException();
+ } catch (NamespaceNotFoundException e) {
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.NAMESPACE_DOESNT_EXIST);
+ }
+ }
+
+ public boolean canExport(TCredentials credentials, String tableId, String tableName, String exportDir, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasTablePermission(credentials, tableId, namespaceId, TablePermission.READ, false);
+ }
+
+ public boolean canImport(TCredentials credentials, String tableName, String importDir, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.CREATE_TABLE, namespaceId, false);
+ }
+
+ public boolean canAlterNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
+ }
+
+ public boolean canCreateNamespace(TCredentials credentials, String namespace) throws ThriftSecurityException {
+ return canCreateNamespace(credentials);
+ }
+
+ private boolean canCreateNamespace(TCredentials credentials) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermission(credentials, SystemPermission.CREATE_NAMESPACE, false);
+ }
+
+ public boolean canDeleteNamespace(TCredentials credentials, String namespaceId) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.DROP_NAMESPACE, namespaceId, false);
+ }
+
+ public boolean canRenameNamespace(TCredentials credentials, String namespaceId, String oldName, String newName) throws ThriftSecurityException {
+ authenticate(credentials);
+ return hasSystemPermissionWithNamespaceId(credentials, SystemPermission.ALTER_NAMESPACE, namespaceId, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 19e7ff7,0000000..872fe11
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@@ -1,149 -1,0 +1,151 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecurityPermission;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Credentials for the system services.
+ *
+ * @since 1.6.0
+ */
+public final class SystemCredentials extends Credentials {
+
+ private static final SecurityPermission SYSTEM_CREDENTIALS_PERMISSION = new SecurityPermission("systemCredentialsPermission");
+
+ private static SystemCredentials SYSTEM_CREDS = null;
+ private static final String SYSTEM_PRINCIPAL = "!SYSTEM";
+
+ private final TCredentials AS_THRIFT;
+
+ SystemCredentials(Instance instance) {
+ super(SYSTEM_PRINCIPAL, SystemToken.get(instance));
+ AS_THRIFT = super.toThrift(instance);
+ }
+
+ public static SystemCredentials get() {
+ check_permission();
+ if (SYSTEM_CREDS == null) {
+ SYSTEM_CREDS = new SystemCredentials(HdfsZooInstance.getInstance());
+ }
+ return SYSTEM_CREDS;
+ }
+
+ private static void check_permission() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(SYSTEM_CREDENTIALS_PERMISSION);
+ }
+ }
+
+ public static SystemCredentials get(Instance instance) {
+ check_permission();
+ /* Special case to avoid duplicating SYSTEM_CREDS */
+ if (null != SYSTEM_CREDS) {
+ if (SYSTEM_CREDS.AS_THRIFT.getInstanceId().equals(instance.getInstanceID())) {
+ return SYSTEM_CREDS;
+ }
+ }
+ return new SystemCredentials(instance);
+ }
+
+ @Override
+ public TCredentials toThrift(Instance instance) {
+ if (!AS_THRIFT.getInstanceId().equals(instance.getInstanceID()))
+ throw new IllegalArgumentException("Unexpected instance used for " + SystemCredentials.class.getSimpleName() + ": " + instance.getInstanceID());
+ return AS_THRIFT;
+ }
+
+ /**
+ * An {@link AuthenticationToken} type for Accumulo servers for inter-server communication.
+ *
+ * @since 1.6.0
+ */
+ public static final class SystemToken extends PasswordToken {
+
+ /**
+ * A Constructor for {@link Writable}.
+ */
+ public SystemToken() {}
+
+ private SystemToken(byte[] systemPassword) {
+ super(systemPassword);
+ }
+
+ private static SystemToken get(Instance instance) {
- byte[] instanceIdBytes = instance.getInstanceID().getBytes(Constants.UTF8);
++ byte[] instanceIdBytes = instance.getInstanceID().getBytes(UTF_8);
+ byte[] confChecksum;
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance(Constants.PW_HASH_ALGORITHM);
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("Failed to compute configuration checksum", e);
+ }
+
+ // seed the config with the version and instance id, so at least it's not empty
- md.update(ServerConstants.WIRE_VERSION.toString().getBytes(Constants.UTF8));
++ md.update(ServerConstants.WIRE_VERSION.toString().getBytes(UTF_8));
+ md.update(instanceIdBytes);
+
+ for (Entry<String,String> entry : ServerConfiguration.getSiteConfiguration()) {
+ // only include instance properties
+ if (entry.getKey().startsWith(Property.INSTANCE_PREFIX.toString())) {
- md.update(entry.getKey().getBytes(Constants.UTF8));
- md.update(entry.getValue().getBytes(Constants.UTF8));
++ md.update(entry.getKey().getBytes(UTF_8));
++ md.update(entry.getValue().getBytes(UTF_8));
+ }
+ }
+ confChecksum = md.digest();
+
+ int wireVersion = ServerConstants.WIRE_VERSION;
+
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(3 * (Integer.SIZE / Byte.SIZE) + instanceIdBytes.length + confChecksum.length);
+ DataOutputStream out = new DataOutputStream(bytes);
+ try {
+ out.write(wireVersion * -1);
+ out.write(instanceIdBytes.length);
+ out.write(instanceIdBytes);
+ out.write(confChecksum.length);
+ out.write(confChecksum);
+ } catch (IOException e) {
+ // this is impossible with ByteArrayOutputStream; crash hard if this happens
+ throw new RuntimeException(e);
+ }
+ return new SystemToken(Base64.encodeBase64(bytes.toByteArray()));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
index 5f6ff71,0000000..fa1632c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
@@@ -1,214 -1,0 +1,216 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+// Utility class for adding all authentication info into ZK
+public final class ZKAuthenticator implements Authenticator {
+ static final Logger log = Logger.getLogger(ZKAuthenticator.class);
+ private static Authenticator zkAuthenticatorInstance = null;
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authenticator getInstance() {
+ if (zkAuthenticatorInstance == null)
+ zkAuthenticatorInstance = new ZKAuthenticator();
+ return zkAuthenticatorInstance;
+ }
+
+ public ZKAuthenticator() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = Constants.ZROOT + "/" + instanceId + "/users";
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException {
+ try {
+ // remove old settings from zookeeper first, if any
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ synchronized (zooCache) {
+ zooCache.clear();
+ if (zoo.exists(ZKUserPath)) {
+ zoo.recursiveDelete(ZKUserPath, NodeMissingPolicy.SKIP);
+ log.info("Removed " + ZKUserPath + "/" + " from zookeeper");
+ }
+
+ // prep parent node of users with root username
- zoo.putPersistentData(ZKUserPath, principal.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(ZKUserPath, principal.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ constructUser(principal, ZKSecurityTool.createPass(token));
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up the user in ZK for the provided user. No checking for existence is done here, it should be done before calling.
+ */
+ private void constructUser(String user, byte[] pass) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPrivatePersistentData(ZKUserPath + "/" + user, pass, NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public Set<String> listUsers() {
+ return new TreeSet<String>(zooCache.getChildren(ZKUserPath));
+ }
+
+ /**
+ * Creates a user with no permissions whatsoever
+ */
+ @Override
+ public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ try {
+ if (!(token instanceof PasswordToken))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
+ PasswordToken pt = (PasswordToken) token;
+ constructUser(principal, ZKSecurityTool.createPass(pt.getPassword()));
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_EXISTS, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().recursiveDelete(ZKUserPath + "/" + user, NodeMissingPolicy.FAIL);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ }
+ }
+
+ @Override
+ public void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ if (!(token instanceof PasswordToken))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
+ PasswordToken pt = (PasswordToken) token;
+ if (userExists(principal)) {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + principal);
+ ZooReaderWriter.getInstance().putPrivatePersistentData(ZKUserPath + "/" + principal, ZKSecurityTool.createPass(pt.getPassword()),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.DEFAULT_SECURITY_ERROR, e);
+ }
+ } else
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
+ }
+
+ /**
+ * Checks if a user exists
+ */
+ @Override
+ public boolean userExists(String user) {
+ return zooCache.get(ZKUserPath + "/" + user) != null;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ if (!(token instanceof PasswordToken))
+ throw new AccumuloSecurityException(principal, SecurityErrorCode.INVALID_TOKEN);
+ PasswordToken pt = (PasswordToken) token;
+ byte[] pass;
+ String zpath = ZKUserPath + "/" + principal;
+ pass = zooCache.get(zpath);
+ boolean result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
+ if (!result) {
+ zooCache.clear(zpath);
+ pass = zooCache.get(zpath);
+ result = ZKSecurityTool.checkPass(pt.getPassword(), pass);
+ }
+ return result;
+ }
+
+ @Override
+ public Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes() {
+ Set<Class<? extends AuthenticationToken>> cs = new HashSet<Class<? extends AuthenticationToken>>();
+ cs.add(PasswordToken.class);
+ return cs;
+ }
+
+ @Override
+ public boolean validTokenClass(String tokenClass) {
+ return tokenClass.equals(PasswordToken.class.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index 75b73fc,0000000..78597ad
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -1,171 -1,0 +1,172 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZKAuthorizor implements Authorizor {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static Authorizor zkAuthorizorInstance = null;
+
+ private final String ZKUserAuths = "/Authorizations";
+
+ private String ZKUserPath;
+ private final ZooCache zooCache;
+
+ public static synchronized Authorizor getInstance() {
+ if (zkAuthorizorInstance == null)
+ zkAuthorizorInstance = new ZKAuthorizor();
+ return zkAuthorizorInstance;
+ }
+
+ public ZKAuthorizor() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ }
+
+ @Override
+ public Authorizations getCachedUserAuthorizations(String user) {
+ byte[] authsBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserAuths);
+ if (authsBytes != null)
+ return ZKSecurityTool.convertAuthorizations(authsBytes);
+ return Authorizations.EMPTY;
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the metadata tables
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
- zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserAuths, ZKSecurityTool.convertAuthorizations(Authorizations.EMPTY), NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserAuths, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserAuths, ZKSecurityTool.convertAuthorizations(authorizations),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
+ Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
+ for (ByteBuffer auth : auths)
+ if (!userauths.contains(auth))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index be4806b,0000000..41f880d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -1,515 -1,0 +1,516 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.security.handler;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+
+/**
+ *
+ */
+public class ZKPermHandler implements PermissionHandler {
+ private static final Logger log = Logger.getLogger(ZKAuthorizor.class);
+ private static PermissionHandler zkPermHandlerInstance = null;
+
+ private String ZKUserPath;
+ private String ZKTablePath;
+ private String ZKNamespacePath;
+ private final ZooCache zooCache;
+ private final String ZKUserSysPerms = "/System";
+ private final String ZKUserTablePerms = "/Tables";
+ private final String ZKUserNamespacePerms = "/Namespaces";
+
+ public static synchronized PermissionHandler getInstance() {
+ if (zkPermHandlerInstance == null)
+ zkPermHandlerInstance = new ZKPermHandler();
+ return zkPermHandlerInstance;
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ ZKUserPath = ZKSecurityTool.getInstancePath(instanceId) + "/users";
+ ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables";
+ ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces";
+ }
+
+ public ZKPermHandler() {
+ zooCache = new ZooCache();
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws TableNotFoundException {
+ byte[] serializedPerms;
+ final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table;
+ zrw.sync(path);
+ serializedPerms = zrw.getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the table was just deleted?
+ try {
+ // check for existence:
+ zrw.getData(ZKTablePath + "/" + table, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new TableNotFoundException(null, table, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertTablePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws NamespaceNotFoundException {
+ byte[] serializedPerms;
+ final ZooReaderWriter zrw = ZooReaderWriter.getInstance();
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace;
+ zrw.sync(path);
+ serializedPerms = zrw.getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ // maybe the namespace was just deleted?
+ try {
+ // check for existence:
+ zrw.getData(ZKNamespacePath + "/" + namespace, null);
+ // it's there, you don't have permission
+ return false;
+ } catch (InterruptedException ex) {
+ log.warn("Unhandled InterruptedException, failing closed for namespace permission check", e);
+ return false;
+ } catch (KeeperException ex) {
+ // not there, throw an informative exception
+ if (e.code() == Code.NONODE) {
+ throw new NamespaceNotFoundException(null, namespace, "while checking permissions");
+ }
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ }
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null) {
+ return ZKSecurityTool.convertNamespacePermissions(serializedPerms).contains(permission);
+ }
+ return false;
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ try {
+ byte[] permBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ Set<SystemPermission> perms;
+ if (permBytes == null) {
+ perms = new TreeSet<SystemPermission>();
+ } else {
+ perms = ZKSecurityTool.convertSystemPermissions(permBytes);
+ }
+
+ if (perms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(perms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ Set<TablePermission> tablePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ if (serializedPerms != null)
+ tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ else
+ tablePerms = new TreeSet<TablePermission>();
+
+ try {
+ if (tablePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ Set<NamespacePermission> namespacePerms;
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ if (serializedPerms != null)
+ namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ else
+ namespacePerms = new TreeSet<NamespacePermission>();
+
+ try {
+ if (namespacePerms.add(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] sysPermBytes = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+
+ // User had no system permission, nothing to revoke.
+ if (sysPermBytes == null)
+ return;
+
+ Set<SystemPermission> sysPerms = ZKSecurityTool.convertSystemPermissions(sysPermBytes);
+
+ try {
+ if (sysPerms.remove(permission)) {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(sysPerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table);
+
+ // User had no table permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<TablePermission> tablePerms = ZKSecurityTool.convertTablePermissions(serializedPerms);
+ try {
+ if (tablePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (tablePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, ZKSecurityTool.convertTablePermissions(tablePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException {
+ byte[] serializedPerms = zooCache.get(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace);
+
+ // User had no namespace permission, nothing to revoke.
+ if (serializedPerms == null)
+ return;
+
+ Set<NamespacePermission> namespacePerms = ZKSecurityTool.convertNamespacePermissions(serializedPerms);
+ try {
+ if (namespacePerms.remove(permission)) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (namespacePerms.size() == 0)
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ else
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, ZKSecurityTool.convertNamespacePermissions(namespacePerms),
+ NodeExistsPolicy.OVERWRITE);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ zooCache.clear();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ for (String user : zooCache.getChildren(ZKUserPath))
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace, NodeMissingPolicy.SKIP);
+ }
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException("unknownUser", SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials itw, String rootuser) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ // create the root user with all system privileges, no table privileges, and no record-level authorizations
+ Set<SystemPermission> rootPerms = new TreeSet<SystemPermission>();
+ for (SystemPermission p : SystemPermission.values())
+ rootPerms.add(p);
+ Map<String,Set<TablePermission>> tablePerms = new HashMap<String,Set<TablePermission>>();
+ // Allow the root user to flush the system tables
+ tablePerms.put(RootTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ tablePerms.put(MetadataTable.ID, Collections.singleton(TablePermission.ALTER_TABLE));
+ // essentially the same but on the system namespace, the ALTER_TABLE permission is now redundant
+ Map<String,Set<NamespacePermission>> namespacePerms = new HashMap<String,Set<NamespacePermission>>();
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_NAMESPACE));
+ namespacePerms.put(Namespaces.ACCUMULO_NAMESPACE_ID, Collections.singleton(NamespacePermission.ALTER_TABLE));
+
+ try {
+ // prep parent node of users with root username
+ if (!zoo.exists(ZKUserPath))
- zoo.putPersistentData(ZKUserPath, rootuser.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(ZKUserPath, rootuser.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ initUser(rootuser);
+ zoo.putPersistentData(ZKUserPath + "/" + rootuser + ZKUserSysPerms, ZKSecurityTool.convertSystemPermissions(rootPerms), NodeExistsPolicy.FAIL);
+ for (Entry<String,Set<TablePermission>> entry : tablePerms.entrySet())
+ createTablePerm(rootuser, entry.getKey(), entry.getValue());
+ for (Entry<String,Set<NamespacePermission>> entry : namespacePerms.entrySet())
+ createNamespacePerm(rootuser, entry.getKey(), entry.getValue());
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.putPersistentData(ZKUserPath + "/" + user, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms, new byte[0], NodeExistsPolicy.SKIP);
+ zoo.putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms, new byte[0], NodeExistsPolicy.SKIP);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sets up a new table configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createTablePerm(String user, String table, Set<TablePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserTablePerms + "/" + table,
+ ZKSecurityTool.convertTablePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ /**
+ * Sets up a new namespace configuration for the provided user/table. No checking for existence is done here, it should be done before calling.
+ */
+ private void createNamespacePerm(String user, String namespace, Set<NamespacePermission> perms) throws KeeperException, InterruptedException {
+ synchronized (zooCache) {
+ zooCache.clear();
+ ZooReaderWriter.getInstance().putPersistentData(ZKUserPath + "/" + user + ZKUserNamespacePerms + "/" + namespace,
+ ZKSecurityTool.convertNamespacePermissions(perms), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ try {
+ synchronized (zooCache) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserSysPerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserTablePerms, NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(ZKUserPath + "/" + user + ZKUserNamespacePerms, NodeMissingPolicy.SKIP);
+ zooCache.clear(ZKUserPath + "/" + user);
+ }
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ throw new RuntimeException(e);
+ } catch (KeeperException e) {
+ log.error(e, e);
+ if (e.code().equals(KeeperException.Code.NONODE))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST, e);
+ throw new AccumuloSecurityException(user, SecurityErrorCode.CONNECTION_ERROR, e);
+
+ }
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms;
+ try {
+ String path = ZKUserPath + "/" + user + ZKUserSysPerms;
+ ZooReaderWriter.getInstance().sync(path);
+ perms = ZooReaderWriter.getInstance().getData(path, null);
+ } catch (KeeperException e) {
+ if (e.code() == Code.NONODE) {
+ return false;
+ }
+ log.warn("Unhandled KeeperException, failing closed for table permission check", e);
+ return false;
+ } catch (InterruptedException e) {
+ log.warn("Unhandled InterruptedException, failing closed for table permission check", e);
+ return false;
+ }
+
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ byte[] perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return ZKSecurityTool.convertSystemPermissions(perms).contains(permission);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator authent, Authorizor author) {
+ return true;
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ // All proper housekeeping is done on delete and permission granting, no work needs to be done here
+ }
+}
[44/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
----------------------------------------------------------------------
diff --cc shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
index 934a41d,0000000..c3af70a
mode 100644,000000..100644
--- a/shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/ShellUtilTest.java
@@@ -1,67 -1,0 +1,67 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.shell.ShellUtil;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.io.Text;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.google.common.collect.ImmutableList;
+
+public class ShellUtilTest {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ // String with 3 lines, with one empty line
+ private static final String FILEDATA = "line1\n\nline2";
+
+ @Test
+ public void testWithoutDecode() throws IOException {
+ File testFile = new File(folder.getRoot(), "testFileNoDecode.txt");
+ FileUtils.writeStringToFile(testFile, FILEDATA);
+ List<Text> output = ShellUtil.scanFile(testFile.getAbsolutePath(), false);
+ assertEquals(ImmutableList.of(new Text("line1"), new Text("line2")), output);
+ }
+
+ @Test
+ public void testWithDecode() throws IOException {
+ File testFile = new File(folder.getRoot(), "testFileWithDecode.txt");
+ FileUtils.writeStringToFile(testFile, FILEDATA);
+ List<Text> output = ShellUtil.scanFile(testFile.getAbsolutePath(), true);
+ assertEquals(
- ImmutableList.of(new Text(Base64.decodeBase64("line1".getBytes(StandardCharsets.UTF_8))), new Text(Base64.decodeBase64("line2".getBytes(StandardCharsets.UTF_8)))),
++ ImmutableList.of(new Text(Base64.decodeBase64("line1".getBytes(UTF_8))), new Text(Base64.decodeBase64("line2".getBytes(UTF_8)))),
+ output);
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testWithMissingFile() throws FileNotFoundException {
+ ShellUtil.scanFile("missingFile.txt", false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
----------------------------------------------------------------------
diff --cc shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
index e5b7394,0000000..0229cee
mode 100644,000000..100644
--- a/shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/format/DeleterFormatterTest.java
@@@ -1,177 -1,0 +1,177 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.format;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
- import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+
+import jline.UnsupportedTerminal;
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.format.DeleterFormatter;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DeleterFormatterTest {
+ DeleterFormatter formatter;
+ Map<Key,Value> data;
+ BatchWriter writer;
+ BatchWriter exceptionWriter;
+ Shell shellState;
+
+ ByteArrayOutputStream baos;
+ ConsoleReader reader;
+
+ SettableInputStream input;
+
+ class SettableInputStream extends InputStream {
+ ByteArrayInputStream bais;
+
+ @Override
+ public int read() throws IOException {
+ return bais.read();
+ }
+
+ public void set(String in) {
- bais = new ByteArrayInputStream(in.getBytes(StandardCharsets.UTF_8));
++ bais = new ByteArrayInputStream(in.getBytes(UTF_8));
+ }
+ };
+
+ @Before
+ public void setUp() throws IOException, MutationsRejectedException {
+ input = new SettableInputStream();
+ baos = new ByteArrayOutputStream();
+
+ MutationsRejectedException mre = createMock(MutationsRejectedException.class);
+
+ writer = createNiceMock(BatchWriter.class);
+ exceptionWriter = createNiceMock(BatchWriter.class);
+ exceptionWriter.close();
+ expectLastCall().andThrow(mre);
+ exceptionWriter.addMutation(anyObject(Mutation.class));
+ expectLastCall().andThrow(mre);
+
+ shellState = createNiceMock(Shell.class);
+
+ reader = new ConsoleReader(input, baos, new UnsupportedTerminal());
+ expect(shellState.getReader()).andReturn(reader).anyTimes();
+
+ replay(writer, exceptionWriter, shellState);
+
+ data = new TreeMap<Key,Value>();
- data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(StandardCharsets.UTF_8)));
++ data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF_8)));
+ }
+
+ @Test
+ public void testEmpty() {
+ formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true);
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testSingle() throws IOException {
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[DELETED]", " r ", "cf", "cq", "value");
+ }
+
+ @Test
+ public void testNo() throws IOException {
+ input.set("no\n");
- data.put(new Key("z"), new Value("v2".getBytes(StandardCharsets.UTF_8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[SKIPPED]", " r ", "cf", "cq", "value");
+
+ assertTrue(formatter.hasNext());
+ }
+
+ @Test
+ public void testNoConfirmation() throws IOException {
+ input.set("");
- data.put(new Key("z"), new Value("v2".getBytes(StandardCharsets.UTF_8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[SKIPPED]", " r ", "cf", "cq", "value");
+
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testYes() throws IOException {
+ input.set("y\nyes\n");
- data.put(new Key("z"), new Value("v2".getBytes(StandardCharsets.UTF_8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ verify("[DELETED]", " r ", "cf", "cq", "value");
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ verify("[DELETED]", " z ", "v2");
+ }
+
+ @Test
+ public void testMutationException() {
+ formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ assertFalse(formatter.hasNext());
+ }
+
+ private void verify(String... chunks) throws IOException {
+ reader.flush();
+
+ String output = baos.toString();
+ for (String chunk : chunks) {
+ assertTrue(output.contains(chunk));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
index 93c2e98,b6f5214..8693d03
--- a/test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
+++ b/test/src/main/java/org/apache/accumulo/test/BulkImportDirectory.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
index abf31d2,10ee8a4..c9d18e1
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
index d9527c5,c52ed29..1e1006b
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
index a73199c,51405db..c8e821b
--- a/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
index a5ac0e8,3c20858..7b373c4
--- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;
@@@ -83,9 -84,9 +84,9 @@@ public class TestBinaryRows
public static void runTest(Connector connector, Opts opts, BatchWriterOpts bwOpts, ScannerOpts scanOpts) throws Exception {
final Text CF = new Text("cf"), CQ = new Text("cq");
- final byte[] CF_BYTES = "cf".getBytes(StandardCharsets.UTF_8), CQ_BYTES = "cq".getBytes(StandardCharsets.UTF_8);
+ final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
- BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+ BatchWriter bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
boolean delete = opts.mode.equals("delete");
for (long i = 0; i < opts.num; i++) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/TestIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestIngest.java
index 0548f4c,c3e77af..7f6c514
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index 133a5a5,16f0b3f..c5fcf34
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
index d6a16df,4404e2b..b986330
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
index 9bf6343,cb32b6e..797413f
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
index f7393d9,9f2ba34..dcc3e49
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousQuery.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.continuous;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Map.Entry;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
index 77f74da,0e71bf3..60154df
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousScanner.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.continuous;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 515d4a3,34ee9d0..049f9b8
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.lang.reflect.Method;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
index 262f7b0,4f88c1b..680ca92
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
index 4758aa0,17b8269..b3aae46
--- a/test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/Histogram.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
index 82d8660,82f028f..cab3126
--- a/test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/PrintScanTimeHistogram.java
@@@ -16,12 -16,14 +16,13 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
- import java.nio.charset.StandardCharsets;
-
import org.apache.log4j.Logger;
public class PrintScanTimeHistogram {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
index 29800b6,8bf2d97..0824948
--- a/test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/TimeBinner.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.InputStreamReader;
- import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
index ca17fc5,11d19c5..dffd6c3
--- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.continuous;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
index 8d5cc16,348fb02..fdc704d
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestReader.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.ObjectOutputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
index 5954171,e304ab6..e1be8e6
--- a/test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CacheTestWriter.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index 3b1aeaf,949fc52..879128f
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.HashMap;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index e404749,72efee3..96eb214
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.performance.metadata;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
index 353a02b,e4ce3f3..93a8f61
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.randomwalk;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
index 6338d29,67438d9..e13e7d2
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
@@@ -16,20 -16,19 +16,20 @@@
*/
package org.apache.accumulo.test.randomwalk.bulk;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.test.randomwalk.Environment;
import org.apache.accumulo.test.randomwalk.State;
public class BulkMinusOne extends BulkImportTest {
- private static final Value negOne = new Value("-1".getBytes(StandardCharsets.UTF_8));
+ private static final Value negOne = new Value("-1".getBytes(UTF_8));
@Override
- protected void runLater(State state) throws Exception {
+ protected void runLater(State state, Environment env) throws Exception {
log.info("Decrementing");
- BulkPlusOne.bulkLoadLots(log, state, negOne);
+ BulkPlusOne.bulkLoadLots(log, state, env, negOne);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
index 3c2be07,b4fa7ba..f7a727a
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/Verify.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.bulk;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
index 9e50b61,3dcff6e..09bf883
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.concurrent;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.List;
import java.util.Properties;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
index eb60971,ade3e38..0a9d3b9
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.randomwalk.concurrent;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Properties;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
index 28da4f1,61d7c61..03f2f39
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ChangeAuthorizations.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.concurrent;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
index 85340f5,b3519b6..c16160e
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.concurrent;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
index a7669f8,7db2f04..6ca524a
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Verify.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.image;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.security.MessageDigest;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
index a1731d1,d53658b..d239495
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.image;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.security.MessageDigest;
import java.util.Properties;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
index 4a0515c,5dd3f8b..c3c91c0
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/Write.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.multitable;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Properties;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
index f9ba5ba,07834b0..ba1893f
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.randomwalk.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.accumulo.core.client.Connector;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
index ef4593c,fa64e92..2499af2
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.security;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Properties;
@@@ -91,8 -91,8 +92,8 @@@ public class TableOp extends Test
}
if (!canRead && !ambiguousZone)
throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName);
- for (Entry<String,Integer> entry : WalkingSecurity.get(state).getAuthsMap().entrySet()) {
+ for (Entry<String,Integer> entry : WalkingSecurity.get(state,env).getAuthsMap().entrySet()) {
- if (auths.contains(entry.getKey().getBytes(StandardCharsets.UTF_8)))
+ if (auths.contains(entry.getKey().getBytes(UTF_8)))
seen = seen - entry.getValue();
}
if (seen != 0 && !ambiguousAuths)
@@@ -143,13 -143,13 +144,13 @@@
break;
}
case WRITE:
- boolean canWrite = WalkingSecurity.get(state).canWrite(WalkingSecurity.get(state).getTabCredentials(), tableName, namespaceName);
- boolean ambiguousZone = WalkingSecurity.get(state).inAmbiguousZone(conn.whoami(), tp);
+ boolean canWrite = WalkingSecurity.get(state,env).canWrite(WalkingSecurity.get(state,env).getTabCredentials(), tableName, namespaceName);
+ boolean ambiguousZone = WalkingSecurity.get(state,env).inAmbiguousZone(conn.whoami(), tp);
- String key = WalkingSecurity.get(state).getLastKey() + "1";
+ String key = WalkingSecurity.get(state,env).getLastKey() + "1";
Mutation m = new Mutation(new Text(key));
- for (String s : WalkingSecurity.get(state).getAuthsArray()) {
+ for (String s : WalkingSecurity.get(state,env).getAuthsArray()) {
- m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(StandardCharsets.UTF_8)));
+ m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(UTF_8)));
}
BatchWriter writer = null;
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
index a0fdd9d,1fc0850..1795434
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.sequential;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Properties;
import org.apache.accumulo.core.client.BatchWriter;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
index 0f04afc,b607e8d..5959105
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.randomwalk.shard;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.PrintStream;
@@@ -162,11 -162,11 +163,11 @@@ public class BulkInsert extends Test
}
}
- private void sort(State state, FileSystem fs, String tableName, String seqFile, String outputDir, String workDir, int maxSplits) throws Exception {
+ private void sort(State state, Environment env, FileSystem fs, String tableName, String seqFile, String outputDir, String workDir, int maxSplits) throws Exception {
- PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDir + "/splits.txt"))), false, StandardCharsets.UTF_8.name());
+ PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDir + "/splits.txt"))), false, UTF_8.name());
- Connector conn = state.getConnector();
+ Connector conn = env.getConnector();
Collection<Text> splits = conn.tableOperations().listSplits(tableName, maxSplits);
for (Text split : splits)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
index f625870,d9bd7f9..f5ede55
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.test.randomwalk.shard;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.InputStreamReader;
- import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
index 76a20a0,b34defe..238a88d
--- a/test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
+++ b/test/src/main/java/org/apache/accumulo/test/scalability/Ingest.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.scalability;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
index d496ded,49b5d70..a173f79
--- a/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.test;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
index 85aec59,0000000..7ebcdd7
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/LargeSplitRowIT.java
@@@ -1,283 -1,0 +1,284 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.AccumuloServerException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LargeSplitRowIT extends ConfigurableMacIT {
+ static private final Logger log = Logger.getLogger(LargeSplitRowIT.class);
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+
+ Map<String,String> siteConfig = new HashMap<String,String>();
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "50ms");
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ // User added split
+ @Test(timeout = 60 * 1000)
+ public void userAddedSplit() throws Exception {
+
+ log.info("User added split");
+
+ // make a table and lower the TABLE_END_ROW_MAX_SIZE property
+ final String tableName = getUniqueNames(1)[0];
+ final Connector conn = getConnector();
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
+
+ // Create a BatchWriter and add a mutation to the table
+ BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("Row");
+ m.put("cf", "cq", "value");
+ batchWriter.addMutation(m);
+ batchWriter.close();
+
+ // Create a split point that is too large to be an end row and fill it with all 'm'
+ SortedSet<Text> partitionKeys = new TreeSet<Text>();
+ byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = 'm';
+ }
+ partitionKeys.add(new Text(data));
+
+ // try to add the split point that is too large, if the split point is created the test fails.
+ try {
+ conn.tableOperations().addSplits(tableName, partitionKeys);
+ Assert.fail();
+ } catch (AccumuloServerException e) {}
+
+ // Make sure that the information that was written to the table before we tried to add the split point is still correct
+ int counter = 0;
+ final Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : scanner) {
+ counter++;
+ Key k = entry.getKey();
+ Assert.assertEquals("Row", k.getRow().toString());
+ Assert.assertEquals("cf", k.getColumnFamily().toString());
+ Assert.assertEquals("cq", k.getColumnQualifier().toString());
+ Assert.assertEquals("value", entry.getValue().toString());
+
+ }
+ // Make sure there is only one line in the table
+ Assert.assertEquals(1, counter);
+ }
+
+ // Test tablet server split with 250 entries with all the same prefix
+ @Test(timeout = 60 * 1000)
+ public void automaticSplitWith250Same() throws Exception {
+ log.info("Automatic with 250 with same prefix");
+
+ // make a table and lower the configure properties
+ final String tableName = getUniqueNames(1)[0];
+ final Connector conn = getConnector();
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
+
+ // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
+ // Fill this key with all m's except the last spot
+ BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
+ for (int i = 0; i < data.length - 1; i++) {
+ data[i] = (byte) 'm';
+ }
+
+ // Make the last place in the key different for every entry added to the table
+ for (int i = 0; i < 250; i++) {
+ data[data.length - 1] = (byte) i;
+ Mutation m = new Mutation(data);
+ m.put("cf", "cq", "value");
+ batchWriter.addMutation(m);
+ }
+ // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
+ batchWriter.close();
+ conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
+ Thread.sleep(500);
+
+ // Make sure all the data that was put in the table is still correct
+ int count = 0;
+ final Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : scanner) {
+ Key k = entry.getKey();
+ data[data.length - 1] = (byte) count;
- String expected = new String(data, StandardCharsets.UTF_8);
++ String expected = new String(data, UTF_8);
+ Assert.assertEquals(expected, k.getRow().toString());
+ Assert.assertEquals("cf", k.getColumnFamily().toString());
+ Assert.assertEquals("cq", k.getColumnQualifier().toString());
+ Assert.assertEquals("value", entry.getValue().toString());
+ count++;
+ }
+ Assert.assertEquals(250, count);
+
+ // Make sure no splits occurred in the table
+ Assert.assertEquals(0, conn.tableOperations().listSplits(tableName).size());
+ }
+
+ // 10 0's; 10 2's; 10 4's... 10 30's etc
+ @Test(timeout = 60 * 1000)
+ public void automaticSplitWithGaps() throws Exception {
+ log.info("Automatic Split With Gaps");
+
+ automaticSplit(30, 2);
+ }
+
+ // 10 0's; 10 1's; 10 2's... 10 15's etc
+ @Test(timeout = 60 * 1000)
+ public void automaticSplitWithoutGaps() throws Exception {
+ log.info("Automatic Split Without Gaps");
+
+ automaticSplit(15, 1);
+ }
+
+ @Test(timeout = 60 * 1000)
+ public void automaticSplitLater() throws Exception {
+ log.info("Split later");
+ automaticSplit(15, 1);
+
+ final Connector conn = getConnector();
+
+ String tableName = new String();
+ java.util.Iterator<String> iterator = conn.tableOperations().list().iterator();
+
+ while (iterator.hasNext()) {
+ String curr = iterator.next();
+ if (!curr.equals("accumulo.metadata") && !curr.equals("accumulo.root")) {
+ tableName = curr;
+ }
+ }
+
+ // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
+ BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ byte data[] = new byte[10];
+
+ // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value
+ for (int j = 15; j < 150; j += 1) {
+ for (int i = 0; i < data.length - 1; i++) {
+ data[i] = (byte) j;
+ }
+
+ for (int i = 0; i < 25; i++) {
+ data[data.length - 1] = (byte) i;
+ Mutation m = new Mutation(data);
+ m.put("cf", "cq", "value");
+ batchWriter.addMutation(m);
+ }
+ }
+ // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
+ batchWriter.close();
+ conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
+
+ // Make sure a split occurs
+ while (conn.tableOperations().listSplits(tableName).size() == 0) {
+ Thread.sleep(250);
+ }
+
+ Assert.assertTrue(0 < conn.tableOperations().listSplits(tableName).size());
+ }
+
+ private void automaticSplit(int max, int spacing) throws Exception {
+ // make a table and lower the configure properties
+ final String tableName = getUniqueNames(1)[0];
+ final Connector conn = getConnector();
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
+
+ // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
+ BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
+
+ // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value
+ for (int j = 0; j < max; j += spacing) {
+ for (int i = 0; i < data.length - 1; i++) {
+ data[i] = (byte) j;
+ }
+
+ for (int i = 0; i < 10; i++) {
+ data[data.length - 1] = (byte) i;
+ Mutation m = new Mutation(data);
+ m.put("cf", "cq", "value");
+ batchWriter.addMutation(m);
+ }
+ }
+ // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
+ batchWriter.close();
+ conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
+ Thread.sleep(500);
+
+ // Make sure all the data that was put in the table is still correct
+ int count = 0;
+ int extra = 10;
+ final Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : scanner) {
+ if (extra == 10) {
+ extra = 0;
+ for (int i = 0; i < data.length - 1; i++) {
+ data[i] = (byte) count;
+ }
+ count += spacing;
+
+ }
+ Key k = entry.getKey();
+ data[data.length - 1] = (byte) extra;
- String expected = new String(data, StandardCharsets.UTF_8);
++ String expected = new String(data, UTF_8);
+ Assert.assertEquals(expected, k.getRow().toString());
+ Assert.assertEquals("cf", k.getColumnFamily().toString());
+ Assert.assertEquals("cq", k.getColumnQualifier().toString());
+ Assert.assertEquals("value", entry.getValue().toString());
+ extra++;
+ }
+ Assert.assertEquals(10, extra);
+ Assert.assertEquals(max, count);
+
+ // Make sure no splits occured in the table
+ Assert.assertEquals(0, conn.tableOperations().listSplits(tableName).size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index e9de27c,8188deb..5e885b5
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.test;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index d58c263,d5c940d..6025cf2
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.test;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
index 5901326,05de342..0d26445
--- a/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collection;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
index 1dfb3ce,9c4492e..311e354
--- a/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
@@@ -16,9 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
- import java.nio.charset.StandardCharsets;
import java.util.EnumSet;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index 3fabf49,688a326..9f00f74
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
index 3de5cda,465936e..6a9171a
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 4f5fc7d,ffe55bd..80ee990
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
index 355a7bf,92bd714..3e5fec4
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.EnumSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
index b4067d3,7e5944e..b987dec
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
index 67ecf10,5b5249b..0cdebeb
--- a/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
index 3877e0e,0578ef4..f7f8700
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
@@@ -16,9 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.*;
- import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
index 02ab695,d35ba9f..ef64edc
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
@@@ -16,9 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertTrue;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
index f0e4643,d77d060..91b19f6
--- a/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.Map.Entry;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 0512998,f4bee97..f4101de
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index 9ae94a3,189a55c..df18ef8
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
index 50267dc,90b881c..3514fb3
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Map.Entry;
import java.util.TreeSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
index 760fdf7,3547b68..d9c3c77
--- a/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index 8deb51d,43d3a86..5223439
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@@ -16,9 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
index 0dd511d,fccc79f..b75cccc
--- a/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java
@@@ -16,9 -16,9 +16,9 @@@
*/
package org.apache.accumulo.test.functional;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
- import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
index be1a45f,f2460cf..e0d66ce
--- a/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/VisibilityIT.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
index 0f4cf5b,0000000..554b08e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java
@@@ -1,114 -1,0 +1,115 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.replication.ReplicationConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+import com.google.common.net.HostAndPort;
+
+/**
+ *
+ */
+public class MultiTserverReplicationIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(MultiTserverReplicationIT.class);
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(2);
+ }
+
+ @Test
+ public void tserverReplicationServicePortsAreAdvertised() throws Exception {
+ // Wait for the cluster to be up
+ Connector conn = getConnector();
+ Instance inst = conn.getInstance();
+
+ // Wait for a tserver to come up to fulfill this request
+ conn.tableOperations().create("foo");
+ Scanner s = conn.createScanner("foo", Authorizations.EMPTY);
+ Assert.assertEquals(0, Iterables.size(s));
+
+ ZooReader zreader = new ZooReader(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut());
+ Set<String> tserverHost = new HashSet<>();
+ tserverHost.addAll(zreader.getChildren(ZooUtil.getRoot(inst) + Constants.ZTSERVERS));
+
+ Set<HostAndPort> replicationServices = new HashSet<>();
+
+ for (String tserver : tserverHost) {
+ try {
+ byte[] portData = zreader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver, null);
- HostAndPort replAddress = HostAndPort.fromString(new String(portData, StandardCharsets.UTF_8));
++ HostAndPort replAddress = HostAndPort.fromString(new String(portData, UTF_8));
+ replicationServices.add(replAddress);
+ } catch (Exception e) {
+ log.error("Could not find port for {}", tserver, e);
+ Assert.fail("Did not find replication port advertisement for " + tserver);
+ }
+ }
+
+ // Each tserver should also have equial replicaiton services running internally
+ Assert.assertEquals("Expected an equal number of replication servicers and tservers", tserverHost.size(), replicationServices.size());
+ }
+
+ @Test
+ public void masterReplicationServicePortsAreAdvertised() throws Exception {
+ // Wait for the cluster to be up
+ Connector conn = getConnector();
+ Instance inst = conn.getInstance();
+
+ // Wait for a tserver to come up to fulfill this request
+ conn.tableOperations().create("foo");
+ Scanner s = conn.createScanner("foo", Authorizations.EMPTY);
+ Assert.assertEquals(0, Iterables.size(s));
+
+ ZooReader zreader = new ZooReader(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut());
+
+ // Should have one master instance
+ Assert.assertEquals(1, inst.getMasterLocations().size());
+
+ // Get the master thrift service addr
+ String masterAddr = Iterables.getOnlyElement(inst.getMasterLocations());
+
+ // Get the master replication coordinator addr
- String replCoordAddr = new String(zreader.getData(ZooUtil.getRoot(inst) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR, null), StandardCharsets.UTF_8);
++ String replCoordAddr = new String(zreader.getData(ZooUtil.getRoot(inst) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR, null), UTF_8);
+
+ // They shouldn't be the same
+ Assert.assertNotEquals(masterAddr, replCoordAddr);
+
+ // Neither should be zero as the port
+ Assert.assertNotEquals(0, HostAndPort.fromString(masterAddr).getPort());
+ Assert.assertNotEquals(0, HostAndPort.fromString(replCoordAddr).getPort());
+ }
+}
[36/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index e8d5bbf,0000000..002389a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@@ -1,399 -1,0 +1,400 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.KeeperException.NotEmptyException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import com.google.common.net.HostAndPort;
+
+public class LiveTServerSet implements Watcher {
+
+ public interface Listener {
+ void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added);
+ }
+
+ private static final Logger log = Logger.getLogger(LiveTServerSet.class);
+
+ private final Listener cback;
+ private final Instance instance;
+ private final AccumuloConfiguration conf;
+ private ZooCache zooCache;
+
+ public class TServerConnection {
+ private final HostAndPort address;
+
+ public TServerConnection(HostAndPort addr) throws TException {
+ address = addr;
+ }
+
+ private String lockString(ZooLock mlock) {
+ return mlock.getLockID().serialize(ZooUtil.getRoot(instance) + Constants.ZMASTER_LOCK);
+ }
+
+ public void assignTablet(ZooLock lock, KeyExtent extent) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.loadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void unloadTablet(ZooLock lock, KeyExtent extent, boolean save) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.unloadTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift(), save);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public TabletServerStatus getTableMap(boolean usePooledConnection) throws TException, ThriftSecurityException {
+
+ if (usePooledConnection == true)
+ throw new UnsupportedOperationException();
+
+ TTransport transport = ThriftUtil.createTransport(address, conf);
+
+ try {
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ return client.getTabletServerStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
+ } finally {
+ if (transport != null)
+ transport.close();
+ }
+ }
+
+ public void halt(ZooLock lock) throws TException, ThriftSecurityException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.halt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void fastHalt(ZooLock lock) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.fastHalt(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void flush(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.flush(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
+ startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void chop(ZooLock lock, KeyExtent extent) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.chop(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void splitTablet(ZooLock lock, KeyExtent extent, Text splitPoint) throws TException, ThriftSecurityException, NotServingTabletException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.splitTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), extent.toThrift(),
+ ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength()));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void flushTablet(ZooLock lock, KeyExtent extent) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.flushTablet(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), extent.toThrift());
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public void compact(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ client.compact(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance), lockString(lock), tableId,
+ startRow == null ? null : ByteBuffer.wrap(startRow), endRow == null ? null : ByteBuffer.wrap(endRow));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ public boolean isActive(long tid) throws TException {
+ TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ try {
+ return client.isActive(Tracer.traceInfo(), tid);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ }
+
+ static class TServerInfo {
+ TServerConnection connection;
+ TServerInstance instance;
+
+ TServerInfo(TServerInstance instance, TServerConnection connection) {
+ this.connection = connection;
+ this.instance = instance;
+ }
+ };
+
+ // The set of active tservers with locks, indexed by their name in zookeeper
+ private Map<String,TServerInfo> current = new HashMap<String,TServerInfo>();
+ // as above, indexed by TServerInstance
+ private Map<TServerInstance,TServerInfo> currentInstances = new HashMap<TServerInstance,TServerInfo>();
+
+ // The set of entries in zookeeper without locks, and the first time each was noticed
+ private Map<String,Long> locklessServers = new HashMap<String,Long>();
+
+ public LiveTServerSet(Instance instance, AccumuloConfiguration conf, Listener cback) {
+ this.cback = cback;
+ this.instance = instance;
+ this.conf = conf;
+
+ }
+
+ public synchronized ZooCache getZooCache() {
+ if (zooCache == null)
+ zooCache = new ZooCache(this);
+ return zooCache;
+ }
+
+ public synchronized void startListeningForTabletServerChanges() {
+ scanServers();
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ scanServers();
+ }
+ }, 0, 5000);
+ }
+
+ public synchronized void scanServers() {
+ try {
+ final Set<TServerInstance> updates = new HashSet<TServerInstance>();
+ final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
+
+ final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+
+ HashSet<String> all = new HashSet<String>(current.keySet());
+ all.addAll(getZooCache().getChildren(path));
+
+ locklessServers.keySet().retainAll(all);
+
+ for (String zPath : all) {
+ checkServer(updates, doomed, path, zPath);
+ }
+
+ // log.debug("Current: " + current.keySet());
+ if (!doomed.isEmpty() || !updates.isEmpty())
+ this.cback.update(this, doomed, updates);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private void deleteServerNode(String serverNode) throws InterruptedException, KeeperException {
+ try {
+ ZooReaderWriter.getInstance().delete(serverNode, -1);
+ } catch (NotEmptyException ex) {
+ // race condition: tserver created the lock after our last check; we'll see it at the next check
+ } catch (NoNodeException nne) {
+ // someone else deleted it
+ }
+ }
+
+ private synchronized void checkServer(final Set<TServerInstance> updates, final Set<TServerInstance> doomed, final String path, final String zPath)
+ throws TException, InterruptedException, KeeperException {
+
+ TServerInfo info = current.get(zPath);
+
+ final String lockPath = path + "/" + zPath;
+ Stat stat = new Stat();
+ byte[] lockData = ZooLock.getLockData(getZooCache(), lockPath, stat);
+
+ if (lockData == null) {
+ if (info != null) {
+ doomed.add(info.instance);
+ current.remove(zPath);
+ currentInstances.remove(info.instance);
+ }
+
+ Long firstSeen = locklessServers.get(zPath);
+ if (firstSeen == null) {
+ locklessServers.put(zPath, System.currentTimeMillis());
+ } else if (System.currentTimeMillis() - firstSeen > 10 * 60 * 1000) {
+ deleteServerNode(path + "/" + zPath);
+ locklessServers.remove(zPath);
+ }
+ } else {
+ locklessServers.remove(zPath);
- ServerServices services = new ServerServices(new String(lockData, Constants.UTF8));
++ ServerServices services = new ServerServices(new String(lockData, UTF_8));
+ HostAndPort client = services.getAddress(ServerServices.Service.TSERV_CLIENT);
+ TServerInstance instance = new TServerInstance(client, stat.getEphemeralOwner());
+
+ if (info == null) {
+ updates.add(instance);
+ TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
+ current.put(zPath, tServerInfo);
+ currentInstances.put(instance, tServerInfo);
+ } else if (!info.instance.equals(instance)) {
+ doomed.add(info.instance);
+ updates.add(instance);
+ TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(client));
+ current.put(zPath, tServerInfo);
+ currentInstances.put(info.instance, tServerInfo);
+ }
+ }
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+
+ // its important that these event are propagated by ZooCache, because this ensures when reading zoocache that is has already processed the event and cleared
+ // relevant nodes before code below reads from zoocache
+
+ if (event.getPath() != null) {
+ if (event.getPath().endsWith(Constants.ZTSERVERS)) {
+ scanServers();
+ } else if (event.getPath().contains(Constants.ZTSERVERS)) {
+ int pos = event.getPath().lastIndexOf('/');
+
+ // do only if ZTSERVER is parent
+ if (pos >= 0 && event.getPath().substring(0, pos).endsWith(Constants.ZTSERVERS)) {
+
+ String server = event.getPath().substring(pos + 1);
+
+ final Set<TServerInstance> updates = new HashSet<TServerInstance>();
+ final Set<TServerInstance> doomed = new HashSet<TServerInstance>();
+
+ final String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+
+ try {
+ checkServer(updates, doomed, path, server);
+ if (!doomed.isEmpty() || !updates.isEmpty())
+ this.cback.update(this, doomed, updates);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }
+ }
+
+ public synchronized TServerConnection getConnection(TServerInstance server) {
+ if (server == null)
+ return null;
+ TServerInfo tServerInfo = currentInstances.get(server);
+ if (tServerInfo == null)
+ return null;
+ return tServerInfo.connection;
+ }
+
+ public synchronized Set<TServerInstance> getCurrentServers() {
+ return new HashSet<TServerInstance>(currentInstances.keySet());
+ }
+
+ public synchronized int size() {
+ return current.size();
+ }
+
+ public synchronized TServerInstance find(String tabletServer) {
+ HostAndPort addr = AddressUtil.parseAddress(tabletServer, false);
+ for (Entry<String,TServerInfo> entry : current.entrySet()) {
+ if (entry.getValue().instance.getLocation().equals(addr))
+ return entry.getValue().instance;
+ }
+ return null;
+ }
+
+ public synchronized void remove(TServerInstance server) {
+ String zPath = null;
+ for (Entry<String,TServerInfo> entry : current.entrySet()) {
+ if (entry.getValue().instance.equals(server)) {
+ zPath = entry.getKey();
+ break;
+ }
+ }
+ if (zPath == null)
+ return;
+ current.remove(zPath);
+ currentInstances.remove(server);
+
+ log.info("Removing zookeeper lock for " + server);
+ String fullpath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + zPath;
+ try {
+ ZooReaderWriter.getInstance().recursiveDelete(fullpath, SKIP);
+ } catch (Exception e) {
+ String msg = "error removing tablet server lock";
+ log.fatal(msg, e);
+ Halt.halt(msg, -1);
+ }
+ getZooCache().clear(fullpath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
index 2f657c4,0000000..568e6f6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
@@@ -1,90 -1,0 +1,91 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.DeadServer;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+public class DeadServerList {
+ private static final Logger log = Logger.getLogger(DeadServerList.class);
+ private final String path;
+
+ public DeadServerList(String path) {
+ this.path = path;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.mkdirs(path);
+ } catch (Exception ex) {
+ log.error("Unable to make parent directories of " + path, ex);
+ }
+ }
+
+ public List<DeadServer> getList() {
+ List<DeadServer> result = new ArrayList<DeadServer>();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ List<String> children = zoo.getChildren(path);
+ if (children != null) {
+ for (String child : children) {
+ Stat stat = new Stat();
+ byte[] data;
+ try {
+ data = zoo.getData(path + "/" + child, stat);
+ } catch (NoNodeException nne) {
+ // Another thread or process can delete child while this loop is running.
+ // We ignore this error since it's harmless if we miss the deleted server
+ // in the dead server list.
+ continue;
+ }
- DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, Constants.UTF8));
++ DeadServer server = new DeadServer(child, stat.getMtime(), new String(data, UTF_8));
+ result.add(server);
+ }
+ }
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ return result;
+ }
+
+ public void delete(String server) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
+ zoo.recursiveDelete(path + "/" + server, NodeMissingPolicy.SKIP);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ public void post(String server, String cause) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ try {
- zoo.putPersistentData(path + "/" + server, cause.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++ zoo.putPersistentData(path + "/" + server, cause.getBytes(UTF_8), NodeExistsPolicy.SKIP);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
index 400d0ac,0000000..b4ef519
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
@@@ -1,143 -1,0 +1,144 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * A tablet is assigned to a tablet server at the given address as long as it is alive and well. When the tablet server is restarted, the instance information
+ * it advertises will change. Therefore tablet assignments can be considered out-of-date if the tablet server instance information has been changed.
+ *
+ */
+public class TServerInstance implements Comparable<TServerInstance>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // HostAndPort is not Serializable
+ private transient HostAndPort location;
+ private String session;
+ private String cachedStringRepresentation;
+
+ public TServerInstance(HostAndPort address, String session) {
+ this.location = address;
+ this.session = session;
+ this.cachedStringRepresentation = hostPort() + "[" + session + "]";
+ }
+
+ public TServerInstance(HostAndPort address, long session) {
+ this(address, Long.toHexString(session));
+ }
+
+ public TServerInstance(String address, long session) {
+ this(AddressUtil.parseAddress(address, false), Long.toHexString(session));
+ }
+
+ public TServerInstance(Value address, Text session) {
- this(AddressUtil.parseAddress(new String(address.get(), Constants.UTF8), false), session.toString());
++ this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString());
+ }
+
+ public void putLocation(Mutation m) {
+ m.put(TabletsSection.CurrentLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
+ }
+
+ public void putFutureLocation(Mutation m) {
+ m.put(TabletsSection.FutureLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
+ }
+
+ public void putLastLocation(Mutation m) {
+ m.put(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier(), asMutationValue());
+ }
+
+ public void clearLastLocation(Mutation m) {
+ m.putDelete(TabletsSection.LastLocationColumnFamily.NAME, asColumnQualifier());
+ }
+
+ @Override
+ public int compareTo(TServerInstance other) {
+ if (this == other)
+ return 0;
+ return this.toString().compareTo(other.toString());
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof TServerInstance) {
+ return compareTo((TServerInstance) obj) == 0;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return cachedStringRepresentation;
+ }
+
+ public int port() {
+ return getLocation().getPort();
+ }
+
+ public String host() {
+ return getLocation().getHostText();
+ }
+
+ public String hostPort() {
+ return getLocation().toString();
+ }
+
+ public Text asColumnQualifier() {
+ return new Text(this.getSession());
+ }
+
+ public Value asMutationValue() {
- return new Value(getLocation().toString().getBytes(Constants.UTF8));
++ return new Value(getLocation().toString().getBytes(UTF_8));
+ }
+
+ public HostAndPort getLocation() {
+ return location;
+ }
+
+ public String getSession() {
+ return session;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.defaultWriteObject();
+ out.writeObject(location.toString());
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ location = HostAndPort.fromString(in.readObject().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
index f4d5591,0000000..91be9c3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
@@@ -1,189 -1,0 +1,190 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SkippingIterator;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+
+public class TabletStateChangeIterator extends SkippingIterator {
+
+ private static final String SERVERS_OPTION = "servers";
+ private static final String TABLES_OPTION = "tables";
+ private static final String MERGES_OPTION = "merges";
+ // private static final Logger log = Logger.getLogger(TabletStateChangeIterator.class);
+
+ Set<TServerInstance> current;
+ Set<String> onlineTables;
+ Map<Text,MergeInfo> merges;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+ current = parseServers(options.get(SERVERS_OPTION));
+ onlineTables = parseTables(options.get(TABLES_OPTION));
+ merges = parseMerges(options.get(MERGES_OPTION));
+ }
+
+ private Set<String> parseTables(String tables) {
+ if (tables == null)
+ return null;
+ Set<String> result = new HashSet<String>();
+ for (String table : tables.split(","))
+ result.add(table);
+ return result;
+ }
+
+ private Set<TServerInstance> parseServers(String servers) {
+ if (servers == null)
+ return null;
+ // parse "host:port[INSTANCE]"
+ Set<TServerInstance> result = new HashSet<TServerInstance>();
+ if (servers.length() > 0) {
+ for (String part : servers.split(",")) {
+ String parts[] = part.split("\\[", 2);
+ String hostport = parts[0];
+ String instance = parts[1];
+ if (instance != null && instance.endsWith("]"))
+ instance = instance.substring(0, instance.length() - 1);
+ result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance));
+ }
+ }
+ return result;
+ }
+
+ private Map<Text,MergeInfo> parseMerges(String merges) {
+ if (merges == null)
+ return null;
+ try {
+ Map<Text,MergeInfo> result = new HashMap<Text,MergeInfo>();
+ DataInputBuffer buffer = new DataInputBuffer();
- byte[] data = Base64.decodeBase64(merges.getBytes(Constants.UTF8));
++ byte[] data = Base64.decodeBase64(merges.getBytes(UTF_8));
+ buffer.reset(data, data.length);
+ while (buffer.available() > 0) {
+ MergeInfo mergeInfo = new MergeInfo();
+ mergeInfo.readFields(buffer);
+ result.put(mergeInfo.extent.getTableId(), mergeInfo);
+ }
+ return result;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ protected void consume() throws IOException {
+ while (getSource().hasTop()) {
+ Key k = getSource().getTopKey();
+ Value v = getSource().getTopValue();
+
+ if (onlineTables == null || current == null)
+ return;
+
+ TabletLocationState tls;
+ try {
+ tls = MetaDataTableScanner.createTabletLocationState(k, v);
+ if (tls == null)
+ return;
+ } catch (BadLocationStateException e) {
+ // maybe the master can do something with a tablet with bad/inconsistent state
+ return;
+ }
+ // we always want data about merges
+ MergeInfo merge = merges.get(tls.extent.getTableId());
+ if (merge != null && merge.getExtent() != null && merge.getExtent().overlaps(tls.extent)) {
+ return;
+ }
+ // is the table supposed to be online or offline?
+ boolean shouldBeOnline = onlineTables.contains(tls.extent.getTableId().toString());
+
+ switch (tls.getState(current)) {
+ case ASSIGNED:
+ // we always want data about assigned tablets
+ return;
+ case HOSTED:
+ if (!shouldBeOnline)
+ return;
+ case ASSIGNED_TO_DEAD_SERVER:
+ return;
+ case UNASSIGNED:
+ if (shouldBeOnline)
+ return;
+ }
+ // table is in the expected state so don't bother returning any information about it
+ getSource().next();
+ }
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ public static void setCurrentServers(IteratorSetting cfg, Set<TServerInstance> goodServers) {
+ if (goodServers != null) {
+ List<String> servers = new ArrayList<String>();
+ for (TServerInstance server : goodServers)
+ servers.add(server.toString());
+ cfg.addOption(SERVERS_OPTION, StringUtil.join(servers, ","));
+ }
+ }
+
+ public static void setOnlineTables(IteratorSetting cfg, Set<String> onlineTables) {
+ if (onlineTables != null)
+ cfg.addOption(TABLES_OPTION, StringUtil.join(onlineTables, ","));
+ }
+
+ public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) {
+ DataOutputBuffer buffer = new DataOutputBuffer();
+ try {
+ for (MergeInfo info : merges) {
+ KeyExtent extent = info.getExtent();
+ if (extent != null && !info.getState().equals(MergeState.NONE)) {
+ info.write(buffer);
+ }
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ String encoded = Base64.encodeBase64String(Arrays.copyOf(buffer.getData(), buffer.getLength()));
+ cfg.addOption(MERGES_OPTION, encoded);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
index b0ed03f,0000000..8f0c9c7
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
@@@ -1,97 -1,0 +1,98 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+
+public class ZooStore implements DistributedStore {
+
+ private static final Logger log = Logger.getLogger(ZooStore.class);
+
+ String basePath;
+
+ ZooCache cache = new ZooCache();
+
+ public ZooStore(String basePath) throws IOException {
+ if (basePath.endsWith("/"))
+ basePath = basePath.substring(0, basePath.length() - 1);
+ this.basePath = basePath;
+ }
+
+ public ZooStore() throws IOException {
+ this(ZooUtil.getRoot(HdfsZooInstance.getInstance().getInstanceID()));
+ }
+
+ @Override
+ public byte[] get(String path) throws DistributedStoreException {
+ try {
+ return cache.get(relative(path));
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ private String relative(String path) {
+ return basePath + path;
+ }
+
+ @Override
+ public List<String> getChildren(String path) throws DistributedStoreException {
+ try {
+ return cache.getChildren(relative(path));
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public void put(String path, byte[] bs) throws DistributedStoreException {
+ try {
+ path = relative(path);
+ ZooReaderWriter.getInstance().putPersistentData(path, bs, NodeExistsPolicy.OVERWRITE);
+ cache.clear();
- log.debug("Wrote " + new String(bs, Constants.UTF8) + " to " + path);
++ log.debug("Wrote " + new String(bs, UTF_8) + " to " + path);
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public void remove(String path) throws DistributedStoreException {
+ try {
+ log.debug("Removing " + path);
+ path = relative(path);
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.exists(path))
+ zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ cache.clear();
+ } catch (Exception ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
index 0a0e1ee,0000000..15ade65
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
@@@ -1,179 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.log4j.Logger;
+
+import com.google.common.net.HostAndPort;
+
+public class ZooTabletStateStore extends TabletStateStore {
+
+ private static final Logger log = Logger.getLogger(ZooTabletStateStore.class);
+ final private DistributedStore store;
+
+ public ZooTabletStateStore(DistributedStore store) {
+ this.store = store;
+ }
+
+ public ZooTabletStateStore() throws DistributedStoreException {
+ try {
+ store = new ZooStore();
+ } catch (IOException ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public ClosableIterator<TabletLocationState> iterator() {
+ return new ClosableIterator<TabletLocationState>() {
+ boolean finished = false;
+
+ @Override
+ public boolean hasNext() {
+ return !finished;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ finished = true;
+ try {
+ byte[] future = store.get(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
+ byte[] current = store.get(RootTable.ZROOT_TABLET_LOCATION);
+ byte[] last = store.get(RootTable.ZROOT_TABLET_LAST_LOCATION);
+
+ TServerInstance currentSession = null;
+ TServerInstance futureSession = null;
+ TServerInstance lastSession = null;
+
+ if (future != null)
+ futureSession = parse(future);
+
+ if (last != null)
+ lastSession = parse(last);
+
+ if (current != null) {
+ currentSession = parse(current);
+ futureSession = null;
+ }
+ List<Collection<String>> logs = new ArrayList<Collection<String>>();
+ for (String entry : store.getChildren(RootTable.ZROOT_TABLET_WALOGS)) {
+ byte[] logInfo = store.get(RootTable.ZROOT_TABLET_WALOGS + "/" + entry);
+ if (logInfo != null) {
+ LogEntry logEntry = new LogEntry();
+ logEntry.fromBytes(logInfo);
+ logs.add(logEntry.logSet);
+ log.debug("root tablet logSet " + logEntry.logSet);
+ }
+ }
+ TabletLocationState result = new TabletLocationState(RootTable.EXTENT, futureSession, currentSession, lastSession, logs, false);
+ log.debug("Returning root tablet state: " + result);
+ return result;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void close() {
+ }
+ };
+ }
+
+ protected TServerInstance parse(byte[] current) {
- String str = new String(current, Constants.UTF8);
++ String str = new String(current, UTF_8);
+ String[] parts = str.split("[|]", 2);
+ HostAndPort address = HostAndPort.fromString(parts[0]);
+ if (parts.length > 1 && parts[1] != null && parts[1].length() > 0) {
+ return new TServerInstance(address, parts[1]);
+ } else {
+ // a 1.2 location specification: DO NOT WANT
+ return null;
+ }
+ }
+
+ @Override
+ public void setFutureLocations(Collection<Assignment> assignments) throws DistributedStoreException {
+ if (assignments.size() != 1)
+ throw new IllegalArgumentException("There is only one root tablet");
+ Assignment assignment = assignments.iterator().next();
+ if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
+ throw new IllegalArgumentException("You can only store the root tablet location");
+ String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
+ Iterator<TabletLocationState> currentIter = iterator();
+ TabletLocationState current = currentIter.next();
+ if (current.current != null) {
+ throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
+ }
- store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(Constants.UTF8));
++ store.put(RootTable.ZROOT_TABLET_FUTURE_LOCATION, value.getBytes(UTF_8));
+ }
+
+ @Override
+ public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException {
+ if (assignments.size() != 1)
+ throw new IllegalArgumentException("There is only one root tablet");
+ Assignment assignment = assignments.iterator().next();
+ if (assignment.tablet.compareTo(RootTable.EXTENT) != 0)
+ throw new IllegalArgumentException("You can only store the root tablet location");
+ String value = assignment.server.getLocation() + "|" + assignment.server.getSession();
+ Iterator<TabletLocationState> currentIter = iterator();
+ TabletLocationState current = currentIter.next();
+ if (current.current != null) {
+ throw new DistributedStoreException("Trying to set the root tablet location: it is already set to " + current.current);
+ }
+ if (!current.future.equals(assignment.server)) {
+ throw new DistributedStoreException("Root tablet is already assigned to " + current.future);
+ }
- store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(Constants.UTF8));
- store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(Constants.UTF8));
++ store.put(RootTable.ZROOT_TABLET_LOCATION, value.getBytes(UTF_8));
++ store.put(RootTable.ZROOT_TABLET_LAST_LOCATION, value.getBytes(UTF_8));
+ // Make the following unnecessary by making the entire update atomic
+ store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
+ log.debug("Put down root tablet location");
+ }
+
+ @Override
+ public void unassign(Collection<TabletLocationState> tablets) throws DistributedStoreException {
+ if (tablets.size() != 1)
+ throw new IllegalArgumentException("There is only one root tablet");
+ TabletLocationState tls = tablets.iterator().next();
+ if (tls.extent.compareTo(RootTable.EXTENT) != 0)
+ throw new IllegalArgumentException("You can only store the root tablet location");
+ store.remove(RootTable.ZROOT_TABLET_LOCATION);
+ store.remove(RootTable.ZROOT_TABLET_FUTURE_LOCATION);
+ log.debug("unassign root tablet location");
+ }
+
+ @Override
+ public String name() {
+ return "Root Table";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 16b4bb8,0000000..409fa62
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -1,273 -1,0 +1,274 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import org.apache.commons.lang.time.DateUtils;
+
+public abstract class AbstractMetricsImpl {
+
+ public class Metric {
+
+ private long count = 0;
+ private long avg = 0;
+ private long min = 0;
+ private long max = 0;
+
+ public long getCount() {
+ return count;
+ }
+
+ public long getAvg() {
+ return avg;
+ }
+
+ public long getMin() {
+ return min;
+ }
+
+ public long getMax() {
+ return max;
+ }
+
+ public void incCount() {
+ count++;
+ }
+
+ public void addAvg(long a) {
+ if (a < 0)
+ return;
+ avg = (long) ((avg * .8) + (a * .2));
+ }
+
+ public void addMin(long a) {
+ if (a < 0)
+ return;
+ min = Math.min(min, a);
+ }
+
+ public void addMax(long a) {
+ if (a < 0)
+ return;
+ max = Math.max(max, a);
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("count", count).append("average", avg).append("minimum", min).append("maximum", max).toString();
+ }
+
+ }
+
+ static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(AbstractMetricsImpl.class);
+
+ private static ConcurrentHashMap<String,Metric> registry = new ConcurrentHashMap<String,Metric>();
+
+ private boolean currentlyLogging = false;
+
+ private File logDir = null;
+
+ private String metricsPrefix = null;
+
+ private Date today = new Date();
+
+ private File logFile = null;
+
+ private Writer logWriter = null;
+
+ private SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+
+ private SimpleDateFormat logFormatter = new SimpleDateFormat("yyyyMMddhhmmssz");
+
+ private MetricsConfiguration config = null;
+
+ public AbstractMetricsImpl() {
+ this.metricsPrefix = getMetricsPrefix();
+ config = new MetricsConfiguration(metricsPrefix);
+ }
+
+ /**
+ * Registers a StandardMBean with the MBean Server
+ */
+ public void register(StandardMBean mbean) throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(mbean, getObjectName());
+
+ setupLogging();
+ }
+
+ /**
+ * Registers this MBean with the MBean Server
+ */
+ public void register() throws Exception {
+ // Register this object with the MBeanServer
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (null == getObjectName())
+ throw new IllegalArgumentException("MBean object name must be set.");
+ mbs.registerMBean(this, getObjectName());
+ setupLogging();
+ }
+
+ public void createMetric(String name) {
+ registry.put(name, new Metric());
+ }
+
+ public Metric getMetric(String name) {
+ return registry.get(name);
+ }
+
+ public long getMetricCount(String name) {
+ return registry.get(name).getCount();
+ }
+
+ public long getMetricAvg(String name) {
+ return registry.get(name).getAvg();
+ }
+
+ public long getMetricMin(String name) {
+ return registry.get(name).getMin();
+ }
+
+ public long getMetricMax(String name) {
+ return registry.get(name).getMax();
+ }
+
+ private void setupLogging() throws IOException {
+ if (null == config.getMetricsConfiguration())
+ return;
+ // If we are already logging, then return
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // Check to see if directory exists, else make it
+ String mDir = config.getMetricsConfiguration().getString("logging.dir");
+ if (null != mDir) {
+ File dir = new File(mDir);
+ if (!dir.isDirectory())
+ if (!dir.mkdir())
+ log.warn("Could not create log directory: " + dir);
+ logDir = dir;
+ // Create new log file
+ startNewLog();
+ }
+ currentlyLogging = true;
+ }
+ }
+
+ private void startNewLog() throws IOException {
+ if (null != logWriter) {
+ logWriter.flush();
+ logWriter.close();
+ }
+ logFile = new File(logDir, metricsPrefix + "-" + formatter.format(today) + ".log");
+ if (!logFile.exists()) {
+ if (!logFile.createNewFile()) {
+ log.error("Unable to create new log file");
+ currentlyLogging = false;
+ return;
+ }
+ }
- logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), Constants.UTF8);
++ logWriter = new OutputStreamWriter(new FileOutputStream(logFile, true), UTF_8);
+ }
+
+ private void writeToLog(String name) throws IOException {
+ if (null == logWriter)
+ return;
+ // Increment the date if we have to
+ Date now = new Date();
+ if (!DateUtils.isSameDay(today, now)) {
+ today = now;
+ startNewLog();
+ }
+ logWriter.append(logFormatter.format(now)).append(" Metric: ").append(name).append(": ").append(registry.get(name).toString()).append("\n");
+ }
+
+ public void add(String name, long time) {
+ if (isEnabled()) {
+ registry.get(name).incCount();
+ registry.get(name).addAvg(time);
+ registry.get(name).addMin(time);
+ registry.get(name).addMax(time);
+ // If we are not currently logging and should be, then initialize
+ if (!currentlyLogging && config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ try {
+ setupLogging();
+ } catch (IOException ioe) {
+ log.error("Error setting up log", ioe);
+ }
+ } else if (currentlyLogging && !config.getMetricsConfiguration().getBoolean(metricsPrefix + ".logging", false)) {
+ // if we are currently logging and shouldn't be, then close logs
+ try {
+ logWriter.flush();
+ logWriter.close();
+ logWriter = null;
+ logFile = null;
+ } catch (Exception e) {
+ log.error("Error stopping metrics logging", e);
+ }
+ currentlyLogging = false;
+ }
+ if (currentlyLogging) {
+ try {
+ writeToLog(name);
+ } catch (IOException ioe) {
+ log.error("Error writing to metrics log", ioe);
+ }
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return config.isEnabled();
+ }
+
+ protected abstract ObjectName getObjectName();
+
+ protected abstract String getMetricsPrefix();
+
+ @Override
+ protected void finalize() {
+ if (null != logWriter) {
+ try {
+ logWriter.close();
+ } catch (Exception e) {
+ // do nothing
+ } finally {
+ logWriter = null;
+ }
+ }
+ logFile = null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
index 0a5341a,0000000..4e909c0
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
@@@ -1,187 -1,0 +1,189 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.monitor;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.net.SocketNode;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Hijack log4j and capture log events for display.
+ *
+ */
+public class LogService extends org.apache.log4j.AppenderSkeleton {
+
+ private static final Logger log = Logger.getLogger(LogService.class);
+
+ /**
+ * Read logging events forward to us over the net.
+ *
+ */
+ static class SocketServer implements Runnable {
+ private ServerSocket server = null;
+
+ public SocketServer(int port) {
+ try {
+ server = new ServerSocket(port);
+ } catch (IOException io) {
+ throw new RuntimeException(io);
+ }
+ }
+
+ public int getLocalPort() {
+ return server.getLocalPort();
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ log.debug("Waiting for log message senders");
+ Socket socket = server.accept();
+ log.debug("Got a new connection");
+ Thread t = new Daemon(new SocketNode(socket, LogManager.getLoggerRepository()));
+ t.start();
+ }
+ } catch (IOException io) {
+ log.error(io, io);
+ }
+ }
+ }
+
+ /**
+ * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
+ *
+ * @param conf
+ * configuration for the instance
+ * @param instanceId
+ * instanceId for the instance
+ * @param hostAddress
+ * Address that monitor process is bound to
+ */
+ public static void startLogListener(AccumuloConfiguration conf, String instanceId, String hostAddress) {
+ try {
+ SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT));
+
+ // getLocalPort will return the actual ephemeral port used when '0' was provided.
+ String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
+
+ log.debug("Setting monitor log4j log-forwarding address to: " + logForwardingAddr);
+
+ final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
+ final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ // Delete before we try to re-create in case the previous session hasn't yet expired
+ try {
+ zoo.delete(path, -1);
+ } catch (KeeperException e) {
+ // We don't care if the node is already gone
+ if (!KeeperException.Code.NONODE.equals(e.code())) {
+ throw e;
+ }
+ }
+
- zoo.putEphemeralData(path, logForwardingAddr.getBytes(Constants.UTF8));
++ zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
+
+ new Daemon(server).start();
+ } catch (Throwable t) {
+ log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
+ }
+ }
+
+ static private LogService instance = null;
+
+ synchronized public static LogService getInstance() {
+ if (instance == null)
+ return new LogService();
+ return instance;
+ }
+
+ private static final int MAX_LOGS = 50;
+
+ private LinkedHashMap<String,DedupedLogEvent> events = new LinkedHashMap<String,DedupedLogEvent>(MAX_LOGS + 1, (float) .75, true) {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ protected boolean removeEldestEntry(Map.Entry eldest) {
+ return size() > MAX_LOGS;
+ }
+ };
+
+ public LogService() {
+ synchronized (LogService.class) {
+ instance = this;
+ }
+ }
+
+ @Override
+ synchronized protected void append(LoggingEvent ev) {
+ Object application = ev.getMDC("application");
+ if (application == null || application.toString().isEmpty())
+ return;
+
+ DedupedLogEvent dev = new DedupedLogEvent(ev);
+
+ // if event is present, increase the count
+ if (events.containsKey(dev.toString())) {
+ DedupedLogEvent oldDev = events.remove(dev.toString());
+ dev.setCount(oldDev.getCount() + 1);
+ }
+ events.put(dev.toString(), dev);
+ }
+
+ @Override
+ public void close() {
+ events = null;
+ }
+
+ @Override
+ public synchronized void doAppend(LoggingEvent event) {
+ super.doAppend(event);
+ }
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ synchronized public List<DedupedLogEvent> getEvents() {
+ return new ArrayList<DedupedLogEvent>(events.values());
+ }
+
+ synchronized public void clear() {
+ events.clear();
+ }
+}
[33/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 03635cc,0000000..9cd02eb
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -1,1051 -1,0 +1,1053 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.BatchWriterImpl;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * provides a reference to the metadata table for updates by tablet servers
+ */
+public class MetadataTableUtil {
+
+ private static final Text EMPTY_TEXT = new Text();
+ private static final byte[] EMPTY_BYTES = new byte[0];
+ private static Map<Credentials,Writer> root_tables = new HashMap<Credentials,Writer>();
+ private static Map<Credentials,Writer> metadata_tables = new HashMap<Credentials,Writer>();
+ private static final Logger log = Logger.getLogger(MetadataTableUtil.class);
+
+ private MetadataTableUtil() {}
+
+ public synchronized static Writer getMetadataTable(Credentials credentials) {
+ Writer metadataTable = metadata_tables.get(credentials);
+ if (metadataTable == null) {
+ metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID);
+ metadata_tables.put(credentials, metadataTable);
+ }
+ return metadataTable;
+ }
+
+ private synchronized static Writer getRootTable(Credentials credentials) {
+ Writer rootTable = root_tables.get(credentials);
+ if (rootTable == null) {
+ rootTable = new Writer(HdfsZooInstance.getInstance(), credentials, RootTable.ID);
+ root_tables.put(credentials, rootTable);
+ }
+ return rootTable;
+ }
+
+ private static void putLockID(ZooLock zooLock, Mutation m) {
+ TabletsSection.ServerColumnFamily.LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/")
- .getBytes(Constants.UTF8)));
++ .getBytes(UTF_8)));
+ }
+
+ private static void update(Credentials credentials, Mutation m, KeyExtent extent) {
+ update(credentials, null, m, extent);
+ }
+
+ public static void update(Credentials credentials, ZooLock zooLock, Mutation m, KeyExtent extent) {
+ Writer t = extent.isMeta() ? getRootTable(credentials) : getMetadataTable(credentials);
+ update(t, credentials, zooLock, m);
+ }
+
+ public static void update(Writer t, Credentials credentials, ZooLock zooLock, Mutation m) {
+ if (zooLock != null)
+ putLockID(zooLock, m);
+ while (true) {
+ try {
+ t.update(m);
+ return;
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ } catch (AccumuloSecurityException e) {
+ log.error(e, e);
+ } catch (ConstraintViolationException e) {
+ log.error(e, e);
+ // retrying when a CVE occurs is probably futile and can cause problems, see ACCUMULO-3096
+ throw new RuntimeException(e);
+ } catch (TableNotFoundException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public static void updateTabletFlushID(KeyExtent extent, long flushID, Credentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(UTF_8)));
+ update(credentials, zooLock, m, extent);
+ }
+ }
+
+ public static void updateTabletCompactID(KeyExtent extent, long compactID, Credentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(UTF_8)));
+ update(credentials, zooLock, m, extent);
+ }
+ }
+
+ public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, Credentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
++ byte[] tidBytes = Long.toString(tid).getBytes(UTF_8);
+
+ for (Entry<FileRef,DataFileValue> entry : estSizes.entrySet()) {
+ Text file = entry.getKey().meta();
+ m.put(DataFileColumnFamily.NAME, file, new Value(entry.getValue().encode()));
+ m.put(TabletsSection.BulkFileColumnFamily.NAME, file, new Value(tidBytes));
+ }
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
+ update(credentials, zooLock, m, extent);
+ }
+
+ public static void updateTabletDir(KeyExtent extent, String newDir, Credentials creds, ZooLock lock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
+ update(creds, lock, m, extent);
+ }
+
+ public static void addTablet(KeyExtent extent, String path, Credentials credentials, char timeType, ZooLock lock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
++ TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(UTF_8)));
+
+ update(credentials, lock, m, extent);
+ }
+
+ public static void updateTabletPrevEndRow(KeyExtent extent, Credentials credentials) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+ update(credentials, m, extent);
+ }
+
+ public static void updateTabletVolumes(KeyExtent extent, List<LogEntry> logsToRemove, List<LogEntry> logsToAdd, List<FileRef> filesToRemove,
+ SortedMap<FileRef,DataFileValue> filesToAdd, String newDir, ZooLock zooLock, Credentials credentials) {
+
+ if (extent.isRootTablet()) {
+ if (newDir != null)
+ throw new IllegalArgumentException("newDir not expected for " + extent);
+
+ if (filesToRemove.size() != 0 || filesToAdd.size() != 0)
+ throw new IllegalArgumentException("files not expected for " + extent);
+
+ // add before removing in case of process death
+ for (LogEntry logEntry : logsToAdd)
+ addLogEntry(credentials, logEntry, zooLock);
+
+ removeUnusedWALEntries(extent, logsToRemove, zooLock);
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+
+ for (LogEntry logEntry : logsToRemove)
+ m.putDelete(logEntry.getColumnFamily(), logEntry.getColumnQualifier());
+
+ for (LogEntry logEntry : logsToAdd)
+ m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
+
+ for (FileRef fileRef : filesToRemove)
+ m.putDelete(DataFileColumnFamily.NAME, fileRef.meta());
+
+ for (Entry<FileRef,DataFileValue> entry : filesToAdd.entrySet())
+ m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
+
+ if (newDir != null)
- ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++ ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
+
+ update(credentials, m, extent);
+ }
+ }
+
+ public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, Credentials credentials) throws IOException {
+ TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
+
+ Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ Text row = extent.getMetadataEntry();
+ VolumeManager fs = VolumeManagerImpl.get();
+
+ Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text(""));
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+
+ mdScanner.setRange(new Range(new Key(row), endKey));
+ for (Entry<Key,Value> entry : mdScanner) {
+
+ if (!entry.getKey().getRow().equals(row))
+ break;
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(new FileRef(fs, entry.getKey()), dfv);
+ }
+
+ return sizes;
+ }
+
+ public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, Credentials credentials, ZooLock zooLock) {
+ KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
+ Mutation m = ke.getPrevRowUpdateMutation();
+ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
+ update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
+ }
+
+ public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, Credentials credentials, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+
- TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
++ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(UTF_8)));
+
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
+ ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
+ update(credentials, zooLock, m, extent);
+ }
+
+ public static void finishSplit(Text metadataEntry, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
+ ZooLock zooLock) {
+ Mutation m = new Mutation(metadataEntry);
+ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
+ ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
+
+ for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
+ }
+
+ for (FileRef pathToRemove : highDatafilesToRemove) {
+ m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta());
+ }
+
+ update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
+ }
+
+ public static void finishSplit(KeyExtent extent, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
+ ZooLock zooLock) {
+ finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
+ }
+
+ public static void addDeleteEntries(KeyExtent extent, Set<FileRef> datafilesToDelete, Credentials credentials) throws IOException {
+
+ String tableId = extent.getTableId().toString();
+
+ // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
+ for (FileRef pathToRemove : datafilesToDelete) {
+ update(credentials, createDeleteMutation(tableId, pathToRemove.path().toString()), extent);
+ }
+ }
+
+ public static void addDeleteEntry(String tableId, String path) throws IOException {
+ update(SystemCredentials.get(), createDeleteMutation(tableId, path), new KeyExtent(new Text(tableId), null, null));
+ }
+
+ public static Mutation createDeleteMutation(String tableId, String pathToRemove) throws IOException {
+ Path path = VolumeManagerImpl.get().getFullPath(tableId, pathToRemove);
+ Mutation delFlag = new Mutation(new Text(MetadataSchema.DeletesSection.getRowPrefix() + path.toString()));
+ delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+ return delFlag;
+ }
+
+ public static void removeScanFiles(KeyExtent extent, Set<FileRef> scanFiles, Credentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+
+ for (FileRef pathToRemove : scanFiles)
+ m.putDelete(ScanFileColumnFamily.NAME, pathToRemove.meta());
+
+ update(credentials, zooLock, m, extent);
+ }
+
+ public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<FileRef,FileUtil.FileInfo> firstAndLastRows,
+ SortedMap<FileRef,DataFileValue> datafiles, SortedMap<FileRef,DataFileValue> lowDatafileSizes, SortedMap<FileRef,DataFileValue> highDatafileSizes,
+ List<FileRef> highDatafilesToRemove) {
+
+ for (Entry<FileRef,DataFileValue> entry : datafiles.entrySet()) {
+
+ Text firstRow = null;
+ Text lastRow = null;
+
+ boolean rowsKnown = false;
+
+ FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
+
+ if (mfi != null) {
+ firstRow = mfi.getFirstRow();
+ lastRow = mfi.getLastRow();
+ rowsKnown = true;
+ }
+
+ if (rowsKnown && firstRow.compareTo(midRow) > 0) {
+ // only in high
+ long highSize = entry.getValue().getSize();
+ long highEntries = entry.getValue().getNumEntries();
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
+ // only in low
+ long lowSize = entry.getValue().getSize();
+ long lowEntries = entry.getValue().getNumEntries();
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
+
+ highDatafilesToRemove.add(entry.getKey());
+ } else {
+ long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
+ long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
+
+ long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
+ long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ }
+ }
+ }
+
+ public static void deleteTable(String tableId, boolean insertDeletes, Credentials credentials, ZooLock lock) throws AccumuloException, IOException {
+ Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
+ Text tableIdText = new Text(tableId);
+ BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000)
+ .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
+
+ // scan metadata for our table and delete everything we find
+ Mutation m = null;
+ ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
+
+ // insert deletes before deleting data from metadata... this makes the code fault tolerant
+ if (insertDeletes) {
+
+ ms.fetchColumnFamily(DataFileColumnFamily.NAME);
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms);
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
+
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ FileRef ref = new FileRef(VolumeManagerImpl.get(), key);
+ bw.addMutation(createDeleteMutation(tableId, ref.meta().toString()));
+ }
+
+ if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
+ bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
+ }
+ }
+
+ bw.flush();
+
+ ms.clearColumns();
+ }
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
+
+ if (m == null) {
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+
+ if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
+ bw.addMutation(m);
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ }
+
+ if (m != null)
+ bw.addMutation(m);
+
+ bw.close();
+ }
+
+ static String getZookeeperLogLocation() {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_WALOGS;
+ }
+
+ public static void addLogEntry(Credentials credentials, LogEntry entry, ZooLock zooLock) {
+ if (entry.extent.isRootTablet()) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID())) {
+ String[] parts = entry.filename.split("/");
+ String uniqueId = parts[parts.length - 1];
+ zoo.putPersistentData(root + "/" + uniqueId, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+ }
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ } catch (IOException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ } else {
+ Mutation m = new Mutation(entry.getRow());
+ m.put(entry.getColumnFamily(), entry.getColumnQualifier(), entry.getValue());
+ update(credentials, zooLock, m, entry.extent);
+ }
+ }
+
+ public static void setRootTabletDir(String dir) throws IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
+ try {
- zoo.putPersistentData(zpath, dir.getBytes(Constants.UTF8), -1, NodeExistsPolicy.OVERWRITE);
++ zoo.putPersistentData(zpath, dir.getBytes(UTF_8), -1, NodeExistsPolicy.OVERWRITE);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ }
+
+ public static String getRootTabletDir() throws IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
+ try {
- return new String(zoo.getData(zpath, null), Constants.UTF8);
++ return new String(zoo.getData(zpath, null), UTF_8);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ }
+
+ public static Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> getFileAndLogEntries(Credentials credentials, KeyExtent extent) throws KeeperException,
+ InterruptedException, IOException {
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ if (extent.isRootTablet()) {
+ getRootLogEntries(result);
+ Path rootDir = new Path(getRootTabletDir());
+ FileStatus[] files = fs.listStatus(rootDir);
+ for (FileStatus fileStatus : files) {
+ if (fileStatus.getPath().toString().endsWith("_tmp")) {
+ continue;
+ }
+ DataFileValue dfv = new DataFileValue(0, 0);
+ sizes.put(new FileRef(fileStatus.getPath().toString(), fileStatus.getPath()), dfv);
+ }
+
+ } else {
+ String systemTableToCheck = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, systemTableToCheck, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.setRange(extent.toMetadataRange());
+
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
+ throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
+ }
+
+ if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
+ result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
+ } else if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(new FileRef(fs, entry.getKey()), dfv);
+ } else {
+ throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
+ }
+ }
+ }
+
+ return new Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>>(result, sizes);
+ }
+
+ public static List<LogEntry> getLogEntries(Credentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
+ log.info("Scanning logging entries for " + extent);
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ if (extent.equals(RootTable.EXTENT)) {
+ log.info("Getting logs for root tablet from zookeeper");
+ getRootLogEntries(result);
+ } else {
+ log.info("Scanning metadata for logs used for tablet " + extent);
+ Scanner scanner = getTabletLogScanner(credentials, extent);
+ Text pattern = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
+ if (row.equals(pattern)) {
+ result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ }
+
+ Collections.sort(result, new Comparator<LogEntry>() {
+ @Override
+ public int compare(LogEntry o1, LogEntry o2) {
+ long diff = o1.timestamp - o2.timestamp;
+ if (diff < 0)
+ return -1;
+ if (diff > 0)
+ return 1;
+ return 0;
+ }
+ });
+ log.info("Returning logs " + result + " for extent " + extent);
+ return result;
+ }
+
+ static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String root = getZookeeperLogLocation();
+ // there's a little race between getting the children and fetching
+ // the data. The log can be removed in between.
+ while (true) {
+ result.clear();
+ for (String child : zoo.getChildren(root)) {
+ LogEntry e = new LogEntry();
+ try {
+ e.fromBytes(zoo.getData(root + "/" + child, null));
+ // upgrade from !0;!0<< -> +r<<
+ e.extent = RootTable.EXTENT;
+ result.add(e);
+ } catch (KeeperException.NoNodeException ex) {
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private static Scanner getTabletLogScanner(Credentials credentials, KeyExtent extent) {
+ String tableId = MetadataTable.ID;
+ if (extent.isMeta())
+ tableId = RootTable.ID;
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, tableId, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ Text start = extent.getMetadataEntry();
+ Key endKey = new Key(start, LogColumnFamily.NAME);
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+ scanner.setRange(new Range(new Key(start), endKey));
+ return scanner;
+ }
+
+ private static class LogEntryIterator implements Iterator<LogEntry> {
+
+ Iterator<LogEntry> zookeeperEntries = null;
+ Iterator<LogEntry> rootTableEntries = null;
+ Iterator<Entry<Key,Value>> metadataEntries = null;
+
+ LogEntryIterator(Credentials creds) throws IOException, KeeperException, InterruptedException {
+ zookeeperEntries = getLogEntries(creds, RootTable.EXTENT).iterator();
+ rootTableEntries = getLogEntries(creds, new KeyExtent(new Text(MetadataTable.ID), null, null)).iterator();
+ try {
+ Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken())
+ .createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ log.info("Setting range to " + MetadataSchema.TabletsSection.getRange());
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ metadataEntries = scanner.iterator();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return zookeeperEntries.hasNext() || rootTableEntries.hasNext() || metadataEntries.hasNext();
+ }
+
+ @Override
+ public LogEntry next() {
+ if (zookeeperEntries.hasNext()) {
+ return zookeeperEntries.next();
+ }
+ if (rootTableEntries.hasNext()) {
+ return rootTableEntries.next();
+ }
+ Entry<Key,Value> entry = metadataEntries.next();
+ return LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static Iterator<LogEntry> getLogEntries(Credentials creds) throws IOException, KeeperException, InterruptedException {
+ return new LogEntryIterator(creds);
+ }
+
+ public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
+ if (extent.isRootTablet()) {
+ for (LogEntry entry : logEntries) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID())) {
+ String parts[] = entry.filename.split("/");
+ zoo.recursiveDelete(root + "/" + parts[parts.length - 1], NodeMissingPolicy.SKIP);
+ }
+ break;
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ for (LogEntry entry : logEntries) {
+ m.putDelete(LogColumnFamily.NAME, new Text(entry.getName()));
+ }
+ update(SystemCredentials.get(), zooLock, m, extent);
+ }
+ }
+
+ private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (srcTableId != null && !cf.startsWith("../") && !cf.contains(":")) {
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ }
+ files.add(cf);
+ }
+ }
+ }
+
+ private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
+
+ KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
+ Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
+
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (!cf.startsWith("../") && !cf.contains(":"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
+ m.put(TabletsSection.LastLocationColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(TabletsSection.LastLocationColumnFamily.NAME)) {
+ // skip
+ } else {
+ m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
+ }
+ }
+ return m;
+ }
+
+ private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
+ String tableName = MetadataTable.NAME;
+ if (tableId.equals(MetadataTable.ID))
+ tableName = RootTable.NAME;
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ mscanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ mscanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
+ mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(mscanner);
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(mscanner);
+ return mscanner;
+ }
+
+ static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator ti;
+ if (srcTableId.equals(MetadataTable.ID))
+ ti = new TabletIterator(createCloneScanner(srcTableId, conn), new Range(), true, true);
+ else
+ ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
+
+ if (!ti.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId);
+
+ while (ti.hasNext())
+ bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
+
+ bw.flush();
+ }
+
+ private static int compareEndRows(Text endRow1, Text endRow2) {
+ return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
+ }
+
+ static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
+ true);
+ TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
+
+ if (!cloneIter.hasNext() || !srcIter.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId + " tableId=" + tableId);
+
+ int rewrites = 0;
+
+ while (cloneIter.hasNext()) {
+ Map<Key,Value> cloneTablet = cloneIter.next();
+ Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ HashSet<String> cloneFiles = new HashSet<String>();
+
+ boolean cloneSuccessful = false;
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(ClonedColumnFamily.NAME)) {
+ cloneSuccessful = true;
+ break;
+ }
+ }
+
+ if (!cloneSuccessful)
+ getFiles(cloneFiles, cloneTablet, null);
+
+ List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
+ Map<Key,Value> srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+
+ Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+
+ int cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
+
+ HashSet<String> srcFiles = new HashSet<String>();
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
+
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+ }
+
+ if (cloneSuccessful)
+ continue;
+
+ if (!srcFiles.containsAll(cloneFiles)) {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
+
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
+ }
+
+ bw.addMutation(m);
+
+ for (Map<Key,Value> st : srcTablets)
+ bw.addMutation(createCloneMutation(srcTableId, tableId, st));
+
+ rewrites++;
+ } else {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
- m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
++ m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+ }
+
+ bw.flush();
+ return rewrites;
+ }
+
+ public static void cloneTable(Instance instance, String srcTableId, String tableId, VolumeManager volumeManager) throws Exception {
+
+ Connector conn = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ while (true) {
+
+ try {
+ initializeClone(srcTableId, tableId, conn, bw);
+
+ // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
+
+ while (true) {
+ int rewrites = checkClone(srcTableId, tableId, conn, bw);
+
+ if (rewrites == 0)
+ break;
+ }
+
+ bw.flush();
+ break;
+
+ } catch (TabletIterator.TabletDeletedException tde) {
+ // tablets were merged in the src table
+ bw.flush();
+
+ // delete what we have cloned and try again
+ deleteTable(tableId, false, SystemCredentials.get(), null);
+
+ log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
+
+ UtilWaitThread.sleep(100);
+ }
+ }
+
+ // delete the clone markers and create directory entries
+ Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
+
+ int dirCount = 0;
+
+ for (Entry<Key,Value> entry : mscanner) {
+ Key k = entry.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
+ String dir = volumeManager.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId
+ + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ }
+
+ public static void chopped(KeyExtent extent, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
++ ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(UTF_8)));
+ update(SystemCredentials.get(), zooLock, m, extent);
+ }
+
+ public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ for (Entry<Key,Value> entry : mscanner) {
+ log.debug("Looking at entry " + entry + " with tid " + tid);
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ log.debug("deleting entry " + entry);
+ Mutation m = new Mutation(entry.getKey().getRow());
+ m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ bw.addMutation(m);
+ }
+ }
+ bw.close();
+ }
+
+ public static List<FileRef> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) throws IOException {
+ List<FileRef> result = new ArrayList<FileRef>();
+ try {
+ VolumeManager fs = VolumeManagerImpl.get();
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY));
+ mscanner.setRange(extent.toMetadataRange());
+ mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ result.add(new FileRef(fs, entry.getKey()));
+ }
+ }
+ return result;
+ } catch (TableNotFoundException ex) {
+ // unlikely
+ throw new RuntimeException("Onos! teh metadata table has vanished!!");
+ }
+ }
+
+ public static Map<FileRef,Long> getBulkFilesLoaded(Credentials credentials, KeyExtent extent) throws IOException {
+ Text metadataRow = extent.getMetadataEntry();
+ Map<FileRef,Long> ret = new HashMap<FileRef,Long>();
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, extent.isMeta() ? RootTable.ID : MetadataTable.ID, Authorizations.EMPTY);
+ scanner.setRange(new Range(metadataRow));
+ scanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : scanner) {
+ Long tid = Long.parseLong(entry.getValue().toString());
+ ret.put(new FileRef(fs, entry.getKey()), tid);
+ }
+ return ret;
+ }
+
+ public static void addBulkLoadInProgressFlag(String path) {
+
+ Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
+ m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+
+ // new KeyExtent is only added to force update to write to the metadata table, not the root table
+ // because bulk loads aren't supported to the metadata table
+ update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
+ }
+
+ public static void removeBulkLoadInProgressFlag(String path) {
+
+ Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+
+ // new KeyExtent is only added to force update to write to the metadata table, not the root table
+ // because bulk loads aren't supported to the metadata table
+ update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
+ }
+
+ /**
+ * During an upgrade we need to move deletion requests for files under the !METADATA table to the root tablet.
+ */
+ public static void moveMetaDeleteMarkers(Instance instance, Credentials creds) {
+ String oldDeletesPrefix = "!!~del";
+ Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false);
+
+ // move old delete markers to new location, to standardize table schema between all metadata tables
+ Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY);
+ scanner.setRange(oldDeletesRange);
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(oldDeletesPrefix)) {
+ moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row, oldDeletesPrefix);
+ } else {
+ break;
+ }
+ }
+
+ }
+
+ public static void moveMetaDeleteMarkersFrom14(Instance instance, Credentials creds) {
+ // new KeyExtent is only added to force update to write to the metadata table, not the root table
+ KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"), null, null);
+
+ // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
+ Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID, Authorizations.EMPTY);
+ scanner.setRange(MetadataSchema.DeletesSection.getRange());
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" + MetadataTable.ID)) {
+ moveDeleteEntry(creds, notMetadata, entry, row, MetadataSchema.DeletesSection.getRowPrefix());
+ } else {
+ break;
+ }
+ }
+ }
+
+ private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent, Entry<Key,Value> entry, String rowID, String prefix) {
+ String filename = rowID.substring(prefix.length());
+
+ // add the new entry first
+ log.info("Moving " + filename + " marker in " + RootTable.NAME);
+ Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
+ m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
+ update(creds, m, RootTable.EXTENT);
+
+ // then remove the old entry
+ m = new Mutation(entry.getKey().getRow());
+ m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
+ update(creds, m, oldExtent);
+ }
+
+ public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
+ TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
+
+ HashSet<ColumnFQ> colSet = null;
+ if (columns != null) {
+ colSet = new HashSet<ColumnFQ>(columns);
+ }
+
+ for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
+
+ if (columns != null && !colSet.contains(new ColumnFQ(entry.getKey()))) {
+ continue;
+ }
+
+ Text row = entry.getKey().getRow();
+
+ SortedMap<ColumnFQ,Value> colVals = tabletEntries.get(row);
+ if (colVals == null) {
+ colVals = new TreeMap<ColumnFQ,Value>();
+ tabletEntries.put(row, colVals);
+ }
+
+ colVals.put(new ColumnFQ(entry.getKey()), entry.getValue());
+ }
+
+ return tabletEntries;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index 4e5df9e,0000000..c0d6894
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+public class NamespacePropUtil {
+ public static boolean setNamespaceProperty(String namespaceId, String property, String value) throws KeeperException, InterruptedException {
+ if (!isPropertyValid(property, value))
+ return false;
+
+ // create the zk node for per-namespace properties for this namespace if it doesn't already exist
+ String zkNamespacePath = getPath(namespaceId);
+ ZooReaderWriter.getInstance().putPersistentData(zkNamespacePath, new byte[0], NodeExistsPolicy.SKIP);
+
+ // create the zk node for this property and set it's data to the specified value
+ String zPath = zkNamespacePath + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ return true;
+ }
+
+ public static boolean isPropertyValid(String property, String value) {
+ Property p = Property.getPropertyByKey(property);
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
+ return false;
+
+ return true;
+ }
+
+ public static void removeNamespaceProperty(String namespaceId, String property) throws InterruptedException, KeeperException {
+ String zPath = getPath(namespaceId) + "/" + property;
+ ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
+ }
+
+ private static String getPath(String namespaceId) {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZNAMESPACES + "/" + namespaceId + Constants.ZNAMESPACE_CONF;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index dd540f2,0000000..8924c85
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -1,131 -1,0 +1,131 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
-
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+
+public class RandomizeVolumes {
+ private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
+
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException {
+ ClientOnRequiredTable opts = new ClientOnRequiredTable();
+ opts.parseArgs(RandomizeVolumes.class.getName(), args);
+ Connector c;
+ if (opts.getToken() == null) {
+ SystemCredentials creds = SystemCredentials.get();
+ c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+ } else {
+ c = opts.getConnector();
+ }
+ try {
+ int status = randomize(c, opts.tableName);
+ System.exit(status);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ System.exit(4);
+ }
+ }
+
+ public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ VolumeManager vm = VolumeManagerImpl.get();
+ if (vm.getVolumes().size() < 2) {
+ log.error("There are not enough volumes configured");
+ return 1;
+ }
+ String tableId = c.tableOperations().tableIdMap().get(tableName);
+ if (null == tableId) {
+ log.error("Could not determine the table ID for table " + tableName);
+ return 2;
+ }
+ TableState tableState = TableManager.getInstance().getTableState(tableId);
+ if (TableState.OFFLINE != tableState) {
+ log.info("Taking " + tableName + " offline");
+ c.tableOperations().offline(tableName, true);
+ log.info(tableName + " offline");
+ }
+ log.info("Rewriting entries for " + tableName);
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ DIRECTORY_COLUMN.fetch(scanner);
+ scanner.setRange(TabletsSection.getRange(tableId));
+ BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
+ int count = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ String oldLocation = entry.getValue().toString();
+ String directory;
+ if (oldLocation.contains(":")) {
+ String[] parts = oldLocation.split(Path.SEPARATOR);
+ String tableIdEntry = parts[parts.length - 2];
+ if (!tableIdEntry.equals(tableId)) {
+ log.error("Unexpected table id found: " + tableIdEntry + ", expected " + tableId + "; skipping");
+ continue;
+ }
+ directory = parts[parts.length - 1];
+ } else {
+ directory = oldLocation.substring(Path.SEPARATOR.length());
+ }
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+
+ String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
- m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(Constants.UTF8)));
++ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
+ if (log.isTraceEnabled()) {
+ log.trace("Replacing " + oldLocation + " with " + newLocation);
+ }
+ writer.addMutation(m);
+ try {
+ vm.mkdirs(new Path(newLocation));
+ } catch (IOException ex) {
+ // nevermind
+ }
+ count++;
+ }
+ writer.close();
+ log.info("Updated " + count + " entries for table " + tableName);
+ if (TableState.OFFLINE != tableState) {
+ c.tableOperations().online(tableName, true);
+ log.info("table " + tableName + " back online");
+ }
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index a08000e,0000000..a82fa14
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,124 -1,0 +1,125 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
- create(root, "", Constants.UTF8.name());
++ create(root, "", UTF_8.name());
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ // assume UTF-8 if not "base64"
+ private void create(String path, String value, String encoding) {
- byte[] data = value.getBytes(Constants.UTF8);
++ byte[] data = value.getBytes(UTF_8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names = "--overwrite")
+ boolean overwrite = false;
+ @Parameter(names = "--file")
+ String file;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index 85f29ba,0000000..b1a553d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,277 -1,0 +1,278 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.Socket;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.SocketFactory;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang.math.LongRange;
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.Filter;
+import org.apache.log4j.spi.LocationInfo;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+import org.apache.log4j.varia.LevelRangeFilter;
+import org.apache.log4j.xml.XMLLayout;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+
+public class SendLogToChainsaw extends XMLLayout {
+
+ private static Pattern logPattern = Pattern.compile(
+ "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
+
+ private File[] logFiles = null;
+
+ private SocketFactory factory = SocketFactory.getDefault();
+
+ private WildcardFileFilter fileFilter = null;
+
+ private Socket socket = null;
+
+ private Pattern lineFilter = null;
+
+ private LongRange dateFilter = null;
+
+ private LevelRangeFilter levelFilter = null;
+
+ public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
+
+ // Set up the file name filter
+ if (null != fileNameFilter) {
+ fileFilter = new WildcardFileFilter(fileNameFilter);
+ } else {
+ fileFilter = new WildcardFileFilter("*");
+ }
+
+ // Get the list of files that match
+ File dir = new File(directory);
+ if (dir.isDirectory()) {
+ logFiles = dir.listFiles((FilenameFilter) fileFilter);
+ } else {
+ throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
+ }
+
+ if (logFiles.length == 0) {
+ throw new IllegalArgumentException("No files match the supplied filter.");
+ }
+
+ socket = factory.createSocket(host, port);
+
+ lineFilter = Pattern.compile(regex);
+
+ // Create Date Filter
+ if (null != start) {
+ if (end == null)
+ end = new Date(System.currentTimeMillis());
+ dateFilter = new LongRange(start.getTime(), end.getTime());
+ }
+
+ if (null != level) {
+ Level base = Level.toLevel(level.toUpperCase());
+ levelFilter = new LevelRangeFilter();
+ levelFilter.setAcceptOnMatch(true);
+ levelFilter.setLevelMin(base);
+ levelFilter.setLevelMax(Level.FATAL);
+ }
+ }
+
+ public void processLogFiles() throws IOException {
+ String line = null;
+ String out = null;
+ InputStreamReader isReader = null;
+ BufferedReader reader = null;
+ try {
+ for (File log : logFiles) {
+ // Parse the server type and name from the log file name
+ String threadName = log.getName().substring(0, log.getName().indexOf("."));
+ try {
- isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
++ isReader = new InputStreamReader(new FileInputStream(log), UTF_8);
+ } catch (FileNotFoundException e) {
+ System.out.println("Unable to find file: " + log.getAbsolutePath());
+ throw e;
+ }
+ reader = new BufferedReader(isReader);
+
+ try {
+ line = reader.readLine();
+ while (null != line) {
+ out = convertLine(line, threadName);
+ if (null != out) {
+ if (socket != null && socket.isConnected())
- socket.getOutputStream().write(out.getBytes(Constants.UTF8));
++ socket.getOutputStream().write(out.getBytes(UTF_8));
+ else
+ System.err.println("Unable to send data to transport");
+ }
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ System.out.println("Error processing line: " + line + ". Output was " + out);
+ throw e;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ if (isReader != null) {
+ isReader.close();
+ }
+ }
+ }
+ } finally {
+ if (socket != null && socket.isConnected()) {
+ socket.close();
+ }
+ }
+ }
+
+ private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
+ String result = null;
+ Matcher m = logPattern.matcher(line);
+ if (m.matches()) {
+
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(new Date(System.currentTimeMillis()));
+ Integer date = Integer.parseInt(m.group(1));
+ Integer hour = Integer.parseInt(m.group(2));
+ Integer min = Integer.parseInt(m.group(3));
+ Integer sec = Integer.parseInt(m.group(4));
+ Integer ms = Integer.parseInt(m.group(5));
+ String clazz = m.group(6);
+ String level = m.group(7);
+ String message = m.group(8);
+ // Apply the regex filter if supplied
+ if (null != lineFilter) {
+ Matcher match = lineFilter.matcher(message);
+ if (!match.matches())
+ return null;
+ }
+ // URL encode the message
- message = URLEncoder.encode(message, Constants.UTF8.name());
++ message = URLEncoder.encode(message, UTF_8.name());
+ // Assume that we are processing logs from today.
+ // If the date in the line is greater than today, then it must be
+ // from the previous month.
+ cal.set(Calendar.DATE, date);
+ cal.set(Calendar.HOUR_OF_DAY, hour);
+ cal.set(Calendar.MINUTE, min);
+ cal.set(Calendar.SECOND, sec);
+ cal.set(Calendar.MILLISECOND, ms);
+ if (date > cal.get(Calendar.DAY_OF_MONTH)) {
+ cal.add(Calendar.MONTH, -1);
+ }
+ long ts = cal.getTimeInMillis();
+ // If this event is not between the start and end dates, then skip it.
+ if (null != dateFilter && !dateFilter.containsLong(ts))
+ return null;
+ Category c = Logger.getLogger(clazz);
+ Level l = Level.toLevel(level);
+ LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
+ (Map<?,?>) null);
+ // Check the log level filter
+ if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
+ return null;
+ }
+ result = format(event);
+ }
+ return result;
+ }
+
+ private static class DateConverter implements IStringConverter<Date> {
+ @Override
+ public Date convert(String value) {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ try {
+ return formatter.parse(value);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private static class Opts extends Help {
+
+ @Parameter(names = {"-d", "--logDirectory"}, description = "ACCUMULO log directory path", required = true)
+ String dir;
+
+ @Parameter(names = {"-f", "--fileFilter"}, description = "filter to apply to names of logs")
+ String filter;
+
+ @Parameter(names = {"-h", "--host"}, description = "host where chainsaw is running", required = true)
+ String hostname;
+
+ @Parameter(names = {"-p", "--port"}, description = "port where XMLSocketReceiver is listening", required = true)
+ int portnum;
+
+ @Parameter(names = {"-s", "--start"}, description = "start date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
+ Date startDate;
+
+ @Parameter(names = {"-e", "--end"}, description = "end date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
+ Date endDate;
+
+ @Parameter(names = {"-l", "--level"}, description = "filter log level")
+ String level;
+
+ @Parameter(names = {"-m", "--messageFilter"}, description = "regex filter for log messages")
+ String regex;
+ }
+
+ /**
+ *
+ * @param args
+ * <ol>
+ * <li>path to log directory</li>
+ * <li>filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive)</li>
+ * <li>chainsaw host</li>
+ * <li>chainsaw port</li>
+ * <li>start date filter</li>
+ * <li>end date filter</li>
+ * <li>optional regex filter to match on each log4j message</li>
+ * <li>optional level filter</li>
+ * </ol>
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(SendLogToChainsaw.class.getName(), args);
+
+ SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
+ c.processLogFiles();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index b6ca527,0000000..401d98c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@@ -1,45 -1,0 +1,47 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+public class SystemPropUtil {
+ public static boolean setSystemProperty(String property, String value) throws KeeperException, InterruptedException {
+ Property p = Property.getPropertyByKey(property);
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property))
+ return false;
+
+ // create the zk node for this property and set it's data to the specified value
+ String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ return true;
+ }
+
+ public static void removeSystemProperty(String property) throws InterruptedException, KeeperException {
+ String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
+ ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.FAIL);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index bcaf9b0,0000000..6ee1593
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+public class TablePropUtil {
+ public static boolean setTableProperty(String tableId, String property, String value) throws KeeperException, InterruptedException {
+ if (!isPropertyValid(property, value))
+ return false;
+
+ // create the zk node for per-table properties for this table if it doesn't already exist
+ String zkTablePath = getTablePath(tableId);
+ ZooReaderWriter.getInstance().putPersistentData(zkTablePath, new byte[0], NodeExistsPolicy.SKIP);
+
+ // create the zk node for this property and set it's data to the specified value
+ String zPath = zkTablePath + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ return true;
+ }
+
+ public static boolean isPropertyValid(String property, String value) {
+ Property p = Property.getPropertyByKey(property);
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
+ return false;
+
+ return true;
+ }
+
+ public static void removeTableProperty(String tableId, String property) throws InterruptedException, KeeperException {
+ String zPath = getTablePath(tableId) + "/" + property;
+ ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
+ }
+
+ private static String getTablePath(String tablename) {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tablename + Constants.ZTABLE_CONF;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 34c2151,0000000..edbb103
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,72 -1,0 +1,74 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
- holder = new String(lockData, Constants.UTF8);
++ holder = new String(lockData, UTF_8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
[32/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index 292129d,0000000..7ae4d5a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@@ -1,150 -1,0 +1,152 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.watcher;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Appender;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.FileWatchdog;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * Watcher that updates the monitor's log4j port from ZooKeeper in a system property
+ */
+public class MonitorLog4jWatcher extends FileWatchdog implements Watcher {
+ private static final Logger log = Logger.getLogger(MonitorLog4jWatcher.class);
+
+ private static final String HOST_PROPERTY_NAME = "org.apache.accumulo.core.host.log";
+ private static final String PORT_PROPERTY_NAME = "org.apache.accumulo.core.host.log.port";
+
+ private final Object lock;
+ private final Log4jConfiguration logConfig;
+ private boolean loggingDisabled = false;
+ protected String path;
+
+ public MonitorLog4jWatcher(String instance, String filename) {
+ super(filename);
+ this.path = ZooUtil.getRoot(instance) + Constants.ZMONITOR_LOG4J_ADDR;
+ this.lock = new Object();
+ this.logConfig = new Log4jConfiguration(filename);
+ doOnChange();
+ }
+
+ boolean isUsingProperties() {
+ return logConfig.isUsingProperties();
+ }
+
+ String getPath() {
+ return path;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // Initially set the logger if the Monitor's log4j advertisement node exists
+ if (ZooReaderWriter.getInstance().exists(path, this))
+ updateMonitorLog4jLocation();
+ log.info("Set watch for Monitor Log4j watcher");
+ } catch (Exception e) {
+ log.error("Unable to set watch for Monitor Log4j watcher on " + path);
+ }
+
+ super.run();
+ }
+
+ @Override
+ public void doOnChange() {
+ // this method gets called in the parent class' constructor
+ // I'm not sure of a better way to get around this. The final modifier helps though.
+ if (null == lock) {
+ return;
+ }
+
+ synchronized (lock) {
+ // We might triggered by file-reloading or from ZK update.
+ // Either way will result in log-forwarding being restarted
+ loggingDisabled = false;
+ log.info("Enabled log-forwarding");
+ logConfig.resetLogger();
+ }
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ // We got an update, process the data in the node
+ updateMonitorLog4jLocation();
+
+ if (event.getPath() != null) {
+ try {
+ ZooReaderWriter.getInstance().exists(event.getPath(), this);
+ } catch (Exception ex) {
+ log.error("Unable to reset watch for Monitor Log4j watcher", ex);
+ }
+ }
+ }
+
+ /**
+ * Read the host and port information for the Monitor's log4j socket and update the system properties so that, on logger refresh, it sees the new information.
+ */
+ protected void updateMonitorLog4jLocation() {
+ try {
- String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), Constants.UTF8);
++ String hostPortString = new String(ZooReaderWriter.getInstance().getData(path, null), UTF_8);
+ HostAndPort hostAndPort = HostAndPort.fromString(hostPortString);
+
+ System.setProperty(HOST_PROPERTY_NAME, hostAndPort.getHostText());
+ System.setProperty(PORT_PROPERTY_NAME, Integer.toString(hostAndPort.getPort()));
+
+ log.info("Changing monitor log4j address to " + hostAndPort.toString());
+
+ doOnChange();
+ } catch (NoNodeException e) {
+ // Not sure on the synchronization guarantees for Loggers and Appenders
+ // on configuration reload
+ synchronized (lock) {
+ // Don't need to try to re-disable'ing it.
+ if (loggingDisabled) {
+ return;
+ }
+
+ Logger logger = LogManager.getLogger("org.apache.accumulo");
+ if (null != logger) {
+ // TODO ACCUMULO-2343 Create a specific appender for log-forwarding to the monitor
+ // that can replace the AsyncAppender+SocketAppender.
+ Appender appender = logger.getAppender("ASYNC");
+ if (null != appender) {
+ log.info("Closing log-forwarding appender");
+ appender.close();
+ log.info("Removing log-forwarding appender");
+ logger.removeAppender(appender);
+ loggingDisabled = true;
+ }
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ log.error("Could not parse host and port information", e);
+ } catch (Exception e) {
+ log.error("Error reading zookeeper data for Monitor Log4j watcher", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
index b02e5d4,0000000..1bbae03
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
@@@ -1,54 -1,0 +1,55 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
+import org.apache.zookeeper.KeeperException;
+
+public class ZooQueueLock extends org.apache.accumulo.fate.zookeeper.ZooQueueLock {
+
+ public ZooQueueLock(String path, boolean ephemeral) throws KeeperException, InterruptedException {
+ super(ZooReaderWriter.getInstance(), path, ephemeral);
+ }
+
+ public static void main(String args[]) throws InterruptedException, KeeperException {
+ ZooQueueLock lock = new ZooQueueLock("/lock", true);
- DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(Constants.UTF8));
- DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(Constants.UTF8));
++ DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(UTF_8));
++ DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(UTF_8));
+ final Lock readLock = rlocker.readLock();
+ readLock.lock();
+ final Lock readLock2 = rlocker.readLock();
+ readLock2.lock();
+ final Lock writeLock = wlocker.writeLock();
+ if (writeLock.tryLock(100, TimeUnit.MILLISECONDS))
+ throw new RuntimeException("Write lock achieved during read lock!");
+ readLock.unlock();
+ readLock2.unlock();
+ writeLock.lock();
+ if (readLock.tryLock(100, TimeUnit.MILLISECONDS))
+ throw new RuntimeException("Read lock achieved during write lock!");
- final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(Constants.UTF8));
++ final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(UTF_8));
+ writeLock2.unlock();
+ readLock.lock();
+ System.out.println("success");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
index 435591d,0000000..c5374a2
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
@@@ -1,42 -1,0 +1,43 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+
+public class ZooReaderWriter extends org.apache.accumulo.fate.zookeeper.ZooReaderWriter {
+ private static final String SCHEME = "digest";
+ private static final String USER = "accumulo";
+ private static ZooReaderWriter instance = null;
+
+ public ZooReaderWriter(String string, int timeInMillis, String secret) {
- super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++ super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
+ }
+
+ public static synchronized ZooReaderWriter getInstance() {
+ if (instance == null) {
+ AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ instance = new ZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
+ conf.get(Property.INSTANCE_SECRET));
+ }
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index 4a7111d,0000000..e15e63e
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@@ -1,64 -1,0 +1,65 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+
+/**
+ * A factory for {@link ZooReaderWriter} objects.
+ */
+public class ZooReaderWriterFactory {
+ private static final String SCHEME = "digest";
+ private static final String USER = "accumulo";
+ private static IZooReaderWriter instance = null;
+
+ /**
+ * Gets a new reader/writer.
+ *
+ * @param string
+ * ZooKeeper connection string
+ * @param timeInMillis
+ * session timeout in milliseconds
+ * @param secret
+ * instance secret
+ * @return reader/writer
+ */
+ public IZooReaderWriter getZooReaderWriter(String string, int timeInMillis, String secret) {
- return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
++ return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF_8));
+ }
+
+ /**
+ * Gets a reader/writer, retrieving ZooKeeper information from the site configuration. The same instance may be returned for multiple calls.
+ *
+ * @return reader/writer
+ */
+ public IZooReaderWriter getInstance() {
+ synchronized (ZooReaderWriterFactory.class) {
+ if (instance == null) {
+ AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ instance = getZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
+ conf.get(Property.INSTANCE_SECRET));
+ }
+ return instance;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index e7f42af,0000000..3aed7ef
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@@ -1,159 -1,0 +1,161 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class NamespaceConfigurationTest {
+ private static final String NSID = "namespace";
+ private static final String ZOOKEEPERS = "localhost";
+ private static final int ZK_SESSION_TIMEOUT = 120000;
+
+ private String iid;
+ private Instance instance;
+ private AccumuloConfiguration parent;
+ private ZooCacheFactory zcf;
+ private ZooCache zc;
+ private NamespaceConfiguration c;
+
+ @Before
+ public void setUp() {
+ iid = UUID.randomUUID().toString();
+ instance = createMock(Instance.class);
+ parent = createMock(AccumuloConfiguration.class);
+ c = new NamespaceConfiguration(NSID, instance, parent);
+ zcf = createMock(ZooCacheFactory.class);
+ c.setZooCacheFactory(zcf);
+
+ expect(instance.getInstanceID()).andReturn(iid);
+ expectLastCall().anyTimes();
+ expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
+ expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
+ replay(instance);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
+ replay(zcf);
+ }
+
+ @Test
+ public void testGetters() {
+ assertEquals(NSID, c.getNamespaceId());
+ assertEquals(parent, c.getParentConfiguration());
+ }
+
+ @Test
+ public void testGet_InZK() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
- "sekrit".getBytes(Constants.UTF8));
++ "sekrit".getBytes(UTF_8));
+ replay(zc);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGet_InParent() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(null);
+ replay(zc);
+ expect(parent.get(p)).andReturn("sekrit");
+ replay(parent);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGet_SkipParentIfAccumuloNS() {
+ c = new NamespaceConfiguration(Namespaces.ACCUMULO_NAMESPACE_ID, instance, parent);
+ c.setZooCacheFactory(zcf);
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + Namespaces.ACCUMULO_NAMESPACE_ID + Constants.ZNAMESPACE_CONF + "/" + p.getKey()))
+ .andReturn(null);
+ replay(zc);
+ assertNull(c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGetProperties() {
+ PropertyFilter filter = new AllFilter();
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ parent.getProperties(props, filter);
+ replay(parent);
+ List<String> children = new java.util.ArrayList<String>();
+ children.add("foo");
+ children.add("ding");
+ expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF)).andReturn(children);
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "foo")).andReturn(
- "bar".getBytes(Constants.UTF8));
++ "bar".getBytes(UTF_8));
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + "ding")).andReturn(
- "dong".getBytes(Constants.UTF8));
++ "dong".getBytes(UTF_8));
+ replay(zc);
+ c.getProperties(props, filter);
+ assertEquals(2, props.size());
+ assertEquals("bar", props.get("foo"));
+ assertEquals("dong", props.get("ding"));
+ }
+
+ @Test
+ public void testObserver() {
+ ConfigurationObserver o = createMock(ConfigurationObserver.class);
+ c.addObserver(o);
+ Collection<ConfigurationObserver> os = c.getObservers();
+ assertEquals(1, os.size());
+ assertTrue(os.contains(o));
+ c.removeObserver(o);
+ os = c.getObservers();
+ assertEquals(0, os.size());
+ }
+
+ @Test
+ public void testInvalidateCache() {
+ // need to do a get so the accessor is created
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZNAMESPACES + "/" + NSID + Constants.ZNAMESPACE_CONF + "/" + p.getKey())).andReturn(
- "sekrit".getBytes(Constants.UTF8));
++ "sekrit".getBytes(UTF_8));
+ zc.clear();
+ replay(zc);
+ c.get(Property.INSTANCE_SECRET);
+ c.invalidateCache();
+ verify(zc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
index 7f07d94,0000000..81148a0
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
@@@ -1,134 -1,0 +1,134 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.endsWith;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ServerConfigurationFactoryTest {
+ private static final String ZK_HOST = "localhost";
+ private static final int ZK_TIMEOUT = 120000;
+ private static final String IID = "iid";
+
+ // use the same mock ZooCacheFactory and ZooCache for all tests
+ private static ZooCacheFactory zcf;
+ private static ZooCache zc;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ zcf = createMock(ZooCacheFactory.class);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache(eq(ZK_HOST), eq(ZK_TIMEOUT), anyObject(NamespaceConfWatcher.class))).andReturn(zc);
+ expectLastCall().anyTimes();
+ expect(zcf.getZooCache(ZK_HOST, ZK_TIMEOUT)).andReturn(zc);
+ expectLastCall().anyTimes();
+ replay(zcf);
+
+ expect(zc.getChildren(anyObject(String.class))).andReturn(null);
+ expectLastCall().anyTimes();
+ // ConfigSanityCheck looks at timeout
- expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(Constants.UTF8));
++ expect(zc.get(endsWith("timeout"))).andReturn(("" + ZK_TIMEOUT + "ms").getBytes(UTF_8));
+ replay(zc);
+ }
+
+ private Instance instance;
+ private ServerConfigurationFactory scf;
+
+ @Before
+ public void setUp() throws Exception {
+ instance = createMock(Instance.class);
+ expect(instance.getInstanceID()).andReturn(IID);
+ expectLastCall().anyTimes();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ServerConfigurationFactory.clearCachedConfigurations();
+ }
+
+ private void mockInstanceForConfig() {
+ expect(instance.getZooKeepers()).andReturn(ZK_HOST);
+ expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_TIMEOUT);
+ }
+
+ private void ready() {
+ replay(instance);
+ scf = new ServerConfigurationFactory(instance);
+ scf.setZooCacheFactory(zcf);
+ }
+
+ @Test
+ public void testGetInstance() {
+ ready();
+ assertSame(instance, scf.getInstance());
+ }
+
+ @Test
+ public void testGetDefaultConfiguration() {
+ ready();
+ DefaultConfiguration c = scf.getDefaultConfiguration();
+ assertNotNull(c);
+ }
+
+ @Test
+ public void testGetSiteConfiguration() {
+ ready();
+ SiteConfiguration c = scf.getSiteConfiguration();
+ assertNotNull(c);
+ }
+
+ @Test
+ public void testGetConfiguration() {
+ mockInstanceForConfig();
+ ready();
+ AccumuloConfiguration c = scf.getConfiguration();
+ assertNotNull(c);
+ }
+
+ private static final String NSID = "NAMESPACE";
+
+ @Test
+ public void testGetNamespaceConfiguration() {
+ mockInstanceForConfig();
+ ready();
+ NamespaceConfiguration c = scf.getNamespaceConfiguration(NSID);
+ assertEquals(NSID, c.getNamespaceId());
+
+ assertSame(c, scf.getNamespaceConfiguration(NSID));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
index 92580f3,0000000..2f38e4c
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
@@@ -1,145 -1,0 +1,146 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TableConfigurationTest {
+ private static final String TID = "table";
+ private static final String ZOOKEEPERS = "localhost";
+ private static final int ZK_SESSION_TIMEOUT = 120000;
+
+ private String iid;
+ private Instance instance;
+ private NamespaceConfiguration parent;
+ private ZooCacheFactory zcf;
+ private ZooCache zc;
+ private TableConfiguration c;
+
+ @Before
+ public void setUp() {
+ iid = UUID.randomUUID().toString();
+ instance = createMock(Instance.class);
+ parent = createMock(NamespaceConfiguration.class);
+ c = new TableConfiguration(iid, instance, TID, parent);
+ zcf = createMock(ZooCacheFactory.class);
+ c.setZooCacheFactory(zcf);
+
+ expect(instance.getInstanceID()).andReturn(iid);
+ expectLastCall().anyTimes();
+ expect(instance.getZooKeepers()).andReturn(ZOOKEEPERS);
+ expect(instance.getZooKeepersSessionTimeOut()).andReturn(ZK_SESSION_TIMEOUT);
+ replay(instance);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache(eq(ZOOKEEPERS), eq(ZK_SESSION_TIMEOUT), anyObject(TableConfWatcher.class))).andReturn(zc);
+ replay(zcf);
+ }
+
+ @Test
+ public void testGetters() {
+ assertEquals(TID, c.getTableId());
+ assertEquals(parent, c.getParentConfiguration());
+ }
+
+ @Test
+ public void testGet_InZK() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
- .andReturn("sekrit".getBytes(Constants.UTF8));
++ .andReturn("sekrit".getBytes(UTF_8));
+ replay(zc);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGet_InParent() {
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(null);
+ replay(zc);
+ expect(parent.get(p)).andReturn("sekrit");
+ replay(parent);
+ assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
+ }
+
+ @Test
+ public void testGetProperties() {
+ PropertyFilter filter = new AllFilter();
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ parent.getProperties(props, filter);
+ replay(parent);
+ List<String> children = new java.util.ArrayList<String>();
+ children.add("foo");
+ children.add("ding");
+ expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF)).andReturn(children);
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(Constants.UTF8));
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(Constants.UTF8));
++ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(UTF_8));
++ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(UTF_8));
+ replay(zc);
+ c.getProperties(props, filter);
+ assertEquals(2, props.size());
+ assertEquals("bar", props.get("foo"));
+ assertEquals("dong", props.get("ding"));
+ }
+
+ @Test
+ public void testObserver() {
+ ConfigurationObserver o = createMock(ConfigurationObserver.class);
+ c.addObserver(o);
+ Collection<ConfigurationObserver> os = c.getObservers();
+ assertEquals(1, os.size());
+ assertTrue(os.contains(o));
+ c.removeObserver(o);
+ os = c.getObservers();
+ assertEquals(0, os.size());
+ }
+
+ @Test
+ public void testInvalidateCache() {
+ // need to do a get so the accessor is created
+ Property p = Property.INSTANCE_SECRET;
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
- .andReturn("sekrit".getBytes(Constants.UTF8));
++ .andReturn("sekrit".getBytes(UTF_8));
+ zc.clear();
+ replay(zc);
+ c.get(Property.INSTANCE_SECRET);
+ c.invalidateCache();
+ verify(zc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
index 1b9d8f7,0000000..4369119
mode 100644,000000..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
@@@ -1,186 -1,0 +1,186 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ZooCachePropertyAccessorTest {
+ private static final String PATH = "/root/path/to/props";
+ private static final Property PROP = Property.INSTANCE_SECRET;
+ private static final String KEY = PROP.getKey();
+ private static final String FULL_PATH = PATH + "/" + KEY;
+ private static final String VALUE = "value";
- private static final byte[] VALUE_BYTES = VALUE.getBytes(Constants.UTF8);
++ private static final byte[] VALUE_BYTES = VALUE.getBytes(UTF_8);
+
+ private ZooCache zc;
+ private ZooCachePropertyAccessor a;
+
+ @Before
+ public void setUp() {
+ zc = createMock(ZooCache.class);
+ a = new ZooCachePropertyAccessor(zc);
+ }
+
+ @Test
+ public void testGetter() {
+ assertSame(zc, a.getZooCache());
+ }
+
+ @Test
+ public void testGet_Valid() {
+ expect(zc.get(FULL_PATH)).andReturn(VALUE_BYTES);
+ replay(zc);
+ assertEquals(VALUE, a.get(PROP, PATH, null));
+ }
+
+ @Test
+ public void testGet_Parent() {
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ expect(parent.get(PROP)).andReturn(VALUE);
+ replay(parent);
+ expect(zc.get(FULL_PATH)).andReturn(null);
+ replay(zc);
+ assertEquals(VALUE, a.get(PROP, PATH, parent));
+ }
+
+ @Test
+ public void testGet_Parent_Null() {
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ expect(parent.get(PROP)).andReturn(null);
+ replay(parent);
+ expect(zc.get(FULL_PATH)).andReturn(null);
+ replay(zc);
+ assertNull(a.get(PROP, PATH, parent));
+ }
+
+ @Test
+ public void testGet_Null_NoParent() {
+ expect(zc.get(FULL_PATH)).andReturn(null);
+ replay(zc);
+ assertNull(a.get(PROP, PATH, null));
+ }
+
+ @Test
+ public void testGet_InvalidFormat() {
+ Property badProp = Property.MASTER_CLIENTPORT;
+ expect(zc.get(PATH + "/" + badProp.getKey())).andReturn(VALUE_BYTES);
+ replay(zc);
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ expect(parent.get(badProp)).andReturn("12345");
+ replay(parent);
+ assertEquals("12345", a.get(badProp, PATH, parent));
+ }
+
+ @Test
+ public void testGetProperties() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ parent.getProperties(props, filter);
+ replay(parent);
+ String child1 = "child1";
+ String child2 = "child2";
+ List<String> children = new java.util.ArrayList<String>();
+ children.add(child1);
+ children.add(child2);
+ expect(zc.getChildren(PATH)).andReturn(children);
+ expect(zc.get(PATH + "/" + child1)).andReturn(VALUE_BYTES);
+ expect(zc.get(PATH + "/" + child2)).andReturn(null);
+ replay(zc);
+ expect(filter.accept(child1)).andReturn(true);
+ expect(filter.accept(child2)).andReturn(true);
+ replay(filter);
+
+ a.getProperties(props, PATH, filter, parent, null);
+ assertEquals(1, props.size());
+ assertEquals(VALUE, props.get(child1));
+ verify(parent);
+ }
+
+ @Test
+ public void testGetProperties_NoChildren() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ parent.getProperties(props, filter);
+ replay(parent);
+ expect(zc.getChildren(PATH)).andReturn(null);
+ replay(zc);
+
+ a.getProperties(props, PATH, filter, parent, null);
+ assertEquals(0, props.size());
+ }
+
+ @Test
+ public void testGetProperties_Filter() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ parent.getProperties(props, filter);
+ replay(parent);
+ String child1 = "child1";
+ List<String> children = new java.util.ArrayList<String>();
+ children.add(child1);
+ expect(zc.getChildren(PATH)).andReturn(children);
+ replay(zc);
+ expect(filter.accept(child1)).andReturn(false);
+ replay(filter);
+
+ a.getProperties(props, PATH, filter, parent, null);
+ assertEquals(0, props.size());
+ }
+
+ @Test
+ public void testGetProperties_ParentFilter() {
+ Map<String,String> props = new java.util.HashMap<String,String>();
+ AccumuloConfiguration parent = createMock(AccumuloConfiguration.class);
+ PropertyFilter filter = createMock(PropertyFilter.class);
+ PropertyFilter parentFilter = createMock(PropertyFilter.class);
+ parent.getProperties(props, parentFilter);
+ replay(parent);
+ expect(zc.getChildren(PATH)).andReturn(null);
+ replay(zc);
+
+ a.getProperties(props, PATH, filter, parent, parentFilter);
+ verify(parent);
+ }
+
+ @Test
+ public void testInvalidateCache() {
+ zc.clear();
+ replay(zc);
+ a.invalidateCache();
+ verify(zc);
+ }
+}
[42/50] [abbrv] git commit: Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
Merge branch '1.5' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9b20a9d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9b20a9d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9b20a9d4
Branch: refs/heads/1.6
Commit: 9b20a9d4062f9ac0d34ce4b2acd7b9b72524a84f
Parents: 5e0f6a4 2deabd3
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Oct 31 23:37:27 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Oct 31 23:37:27 2014 -0400
----------------------------------------------------------------------
assemble/bin/bootstrap_config.sh | 2 +-
.../org/apache/accumulo/core/Constants.java | 13 +++++-
.../apache/accumulo/core/cli/ClientOpts.java | 6 ++-
.../accumulo/core/client/BatchWriterConfig.java | 11 ++---
.../accumulo/core/client/ZooKeeperInstance.java | 14 ++++---
.../client/impl/InstanceOperationsImpl.java | 4 +-
.../accumulo/core/client/impl/Namespaces.java | 6 ++-
.../accumulo/core/client/impl/ServerClient.java | 4 +-
.../core/client/impl/TableOperationsImpl.java | 32 ++++++++-------
.../accumulo/core/client/impl/Tables.java | 10 +++--
.../accumulo/core/client/impl/Writer.java | 5 ++-
.../core/client/lexicoder/StringLexicoder.java | 6 +--
.../core/client/mapreduce/RangeInputSplit.java | 7 ++--
.../mapreduce/lib/impl/ConfiguratorBase.java | 4 +-
.../mapreduce/lib/impl/InputConfigurator.java | 8 ++--
.../mapreduce/lib/impl/OutputConfigurator.java | 7 ++--
.../lib/partition/RangePartitioner.java | 7 ++--
.../accumulo/core/client/mock/MockShell.java | 11 ++---
.../client/security/tokens/PasswordToken.java | 5 ++-
.../accumulo/core/conf/ConfigurationDocGen.java | 9 ++--
.../accumulo/core/data/ArrayByteSequence.java | 8 ++--
.../org/apache/accumulo/core/data/Column.java | 6 +--
.../apache/accumulo/core/data/Condition.java | 9 ++--
.../org/apache/accumulo/core/data/Value.java | 4 +-
.../accumulo/core/file/BloomFilterLayer.java | 7 ++--
.../core/iterators/ColumnFamilyCounter.java | 5 ++-
.../accumulo/core/iterators/LongCombiner.java | 7 ++--
.../accumulo/core/iterators/conf/ColumnSet.java | 7 ++--
.../core/iterators/user/BigDecimalCombiner.java | 7 ++--
.../core/iterators/user/GrepIterator.java | 5 ++-
.../iterators/user/IntersectingIterator.java | 7 ++--
.../core/iterators/user/LargeRowFilter.java | 5 ++-
.../core/iterators/user/RegExFilter.java | 7 ++--
.../iterators/user/RowDeletingIterator.java | 5 ++-
.../iterators/user/SummingArrayCombiner.java | 7 ++--
.../iterators/user/TransformingIterator.java | 7 ++--
.../core/iterators/user/VisibilityFilter.java | 5 ++-
.../accumulo/core/security/Authorizations.java | 13 +++---
.../core/security/ColumnVisibility.java | 43 ++++++++++----------
.../accumulo/core/security/Credentials.java | 11 ++---
.../core/security/VisibilityConstraint.java | 5 ++-
.../core/security/VisibilityParseException.java | 6 +--
.../core/trace/InstanceUserPassword.java | 5 ++-
.../accumulo/core/trace/ZooTraceClient.java | 5 ++-
.../core/util/ByteArrayBackedCharSequence.java | 5 ++-
.../apache/accumulo/core/util/ByteArraySet.java | 6 +--
.../accumulo/core/util/ByteBufferUtil.java | 5 ++-
.../apache/accumulo/core/util/CreateToken.java | 5 ++-
.../org/apache/accumulo/core/util/Encoding.java | 5 ++-
.../apache/accumulo/core/util/FastFormat.java | 4 +-
.../org/apache/accumulo/core/util/Merge.java | 5 ++-
.../apache/accumulo/core/util/MonitorUtil.java | 4 +-
.../org/apache/accumulo/core/util/TextUtil.java | 4 +-
.../apache/accumulo/core/util/shell/Shell.java | 6 ++-
.../util/shell/commands/AddSplitsCommand.java | 7 ++--
.../shell/commands/AuthenticateCommand.java | 5 ++-
.../util/shell/commands/CreateTableCommand.java | 7 ++--
.../util/shell/commands/ExecfileCommand.java | 5 ++-
.../core/util/shell/commands/HiddenCommand.java | 5 ++-
.../core/util/shell/commands/PasswdCommand.java | 5 ++-
.../shell/commands/QuotedStringTokenizer.java | 5 ++-
.../core/util/shell/commands/UserCommand.java | 5 ++-
.../core/client/BatchWriterConfigTest.java | 6 +--
.../core/client/ZooKeeperInstanceTest.java | 9 ++--
.../core/client/mock/MockNamespacesTest.java | 4 +-
.../tokens/CredentialProviderTokenTest.java | 7 ++--
.../security/tokens/PasswordTokenTest.java | 7 ++--
.../accumulo/core/data/ConditionTest.java | 12 +++---
.../core/data/ConditionalMutationTest.java | 10 ++---
.../apache/accumulo/core/data/ValueTest.java | 5 ++-
.../core/iterators/user/RegExFilterTest.java | 7 ++--
.../core/security/ColumnVisibilityTest.java | 6 +--
.../accumulo/core/security/CredentialsTest.java | 6 +--
.../core/security/VisibilityConstraintTest.java | 4 +-
.../security/crypto/BlockedIOStreamTest.java | 10 ++---
.../core/util/format/DeleterFormatterTest.java | 12 +++---
.../ShardedTableDistributionFormatterTest.java | 8 ++--
.../org/apache/accumulo/fate/AdminUtil.java | 8 ++--
.../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++----
.../zookeeper/DistributedReadWriteLock.java | 18 ++++----
.../accumulo/fate/zookeeper/ZooCache.java | 8 ++--
.../apache/accumulo/fate/zookeeper/ZooLock.java | 6 +--
.../accumulo/fate/zookeeper/ZooReservation.java | 9 ++--
.../accumulo/fate/zookeeper/ZooSession.java | 6 +--
.../impl/MiniAccumuloClusterImpl.java | 4 +-
.../org/apache/accumulo/proxy/ProxyServer.java | 7 ++--
.../apache/accumulo/proxy/TestProxyClient.java | 15 +++----
.../java/org/apache/accumulo/proxy/Util.java | 5 ++-
.../org/apache/accumulo/server/Accumulo.java | 4 +-
.../accumulo/server/client/HdfsZooInstance.java | 10 +++--
.../server/conf/ZooCachePropertyAccessor.java | 5 ++-
.../accumulo/server/conf/ZooConfiguration.java | 4 +-
.../server/constraints/MetadataConstraints.java | 14 ++++---
.../apache/accumulo/server/init/Initialize.java | 15 +++----
.../accumulo/server/master/LiveTServerSet.java | 3 +-
.../server/master/state/DeadServerList.java | 7 ++--
.../server/master/state/TServerInstance.java | 7 ++--
.../master/state/TabletStateChangeIterator.java | 5 ++-
.../accumulo/server/master/state/ZooStore.java | 5 ++-
.../master/state/ZooTabletStateStore.java | 11 ++---
.../server/metrics/AbstractMetricsImpl.java | 5 ++-
.../accumulo/server/monitor/LogService.java | 4 +-
.../server/security/SecurityOperation.java | 4 +-
.../server/security/SystemCredentials.java | 10 +++--
.../security/handler/ZKAuthenticator.java | 4 +-
.../server/security/handler/ZKAuthorizor.java | 5 ++-
.../server/security/handler/ZKPermHandler.java | 5 ++-
.../accumulo/server/tables/TableManager.java | 18 ++++----
.../server/tablets/UniqueNameAllocator.java | 10 +++--
.../server/util/AddFilesWithMissingEntries.java | 5 ++-
.../accumulo/server/util/ChangeSecret.java | 5 ++-
.../accumulo/server/util/CleanZookeeper.java | 6 ++-
.../accumulo/server/util/DeleteZooInstance.java | 6 ++-
.../accumulo/server/util/DumpZookeeper.java | 5 ++-
.../accumulo/server/util/ListInstances.java | 6 ++-
.../accumulo/server/util/MetadataTableUtil.java | 32 ++++++++-------
.../accumulo/server/util/NamespacePropUtil.java | 4 +-
.../accumulo/server/util/RandomizeVolumes.java | 6 +--
.../accumulo/server/util/RestoreZookeeper.java | 7 ++--
.../accumulo/server/util/SendLogToChainsaw.java | 9 ++--
.../accumulo/server/util/SystemPropUtil.java | 4 +-
.../accumulo/server/util/TablePropUtil.java | 4 +-
.../accumulo/server/util/TabletServerLocks.java | 4 +-
.../server/watcher/MonitorLog4jWatcher.java | 4 +-
.../accumulo/server/zookeeper/ZooQueueLock.java | 9 ++--
.../server/zookeeper/ZooReaderWriter.java | 5 ++-
.../zookeeper/ZooReaderWriterFactory.java | 5 ++-
.../server/conf/NamespaceConfigurationTest.java | 10 +++--
.../conf/ServerConfigurationFactoryTest.java | 4 +-
.../server/conf/TableConfigurationTest.java | 9 ++--
.../conf/ZooCachePropertyAccessorTest.java | 4 +-
.../java/org/apache/accumulo/master/Master.java | 8 ++--
.../master/recovery/RecoveryManager.java | 4 +-
.../accumulo/master/state/SetGoalState.java | 4 +-
.../accumulo/master/tableOps/BulkImport.java | 8 ++--
.../master/tableOps/CancelCompactions.java | 10 +++--
.../accumulo/master/tableOps/CompactRange.java | 14 ++++---
.../accumulo/master/tableOps/ExportTable.java | 8 ++--
.../accumulo/master/tableOps/ImportTable.java | 16 ++++----
.../accumulo/master/tableOps/RenameTable.java | 6 ++-
.../apache/accumulo/master/tableOps/Utils.java | 14 ++++---
.../master/tserverOps/ShutdownTServer.java | 4 +-
.../org/apache/accumulo/monitor/Monitor.java | 6 ++-
.../accumulo/monitor/ZooKeeperStatus.java | 7 ++--
.../accumulo/monitor/servlets/BasicServlet.java | 10 +++--
.../monitor/servlets/DefaultServlet.java | 5 ++-
.../accumulo/monitor/servlets/ShellServlet.java | 5 ++-
.../accumulo/monitor/servlets/XMLServlet.java | 7 ++--
.../accumulo/monitor/servlets/trace/Basic.java | 5 ++-
.../org/apache/accumulo/tracer/TraceServer.java | 8 ++--
.../tserver/BulkFailedCopyProcessor.java | 5 ++-
.../org/apache/accumulo/tserver/Tablet.java | 10 +++--
.../apache/accumulo/tserver/TabletServer.java | 9 ++--
.../apache/accumulo/tserver/log/DfsLogger.java | 4 +-
.../accumulo/tserver/logger/LogFileValue.java | 11 ++---
.../accumulo/tserver/logger/LogReader.java | 5 ++-
.../accumulo/test/BulkImportDirectory.java | 5 ++-
.../accumulo/test/NativeMapConcurrencyTest.java | 5 ++-
.../accumulo/test/NativeMapPerformanceTest.java | 7 ++--
.../accumulo/test/NativeMapStressTest.java | 5 ++-
.../apache/accumulo/test/TestBinaryRows.java | 7 ++--
.../org/apache/accumulo/test/TestIngest.java | 7 ++--
.../accumulo/test/TestMultiTableIngest.java | 5 ++-
.../test/continuous/ContinuousIngest.java | 8 ++--
.../test/continuous/ContinuousMoru.java | 7 ++--
.../test/continuous/ContinuousQuery.java | 5 ++-
.../test/continuous/ContinuousScanner.java | 5 ++-
.../test/continuous/ContinuousVerify.java | 7 ++--
.../test/continuous/ContinuousWalk.java | 17 ++++----
.../accumulo/test/continuous/Histogram.java | 6 +--
.../test/continuous/PrintScanTimeHistogram.java | 5 ++-
.../accumulo/test/continuous/TimeBinner.java | 5 ++-
.../test/continuous/UndefinedAnalyzer.java | 9 ++--
.../test/functional/CacheTestReader.java | 7 ++--
.../test/functional/CacheTestWriter.java | 11 ++---
.../accumulo/test/functional/ZombieTServer.java | 4 +-
.../metadata/MetadataBatchScanTest.java | 7 ++--
.../apache/accumulo/test/randomwalk/Module.java | 7 ++--
.../test/randomwalk/bulk/BulkMinusOne.java | 5 ++-
.../accumulo/test/randomwalk/bulk/Verify.java | 5 ++-
.../test/randomwalk/concurrent/BatchWrite.java | 5 ++-
.../test/randomwalk/concurrent/BulkImport.java | 5 ++-
.../concurrent/ChangeAuthorizations.java | 7 ++--
.../randomwalk/concurrent/StopTabletServer.java | 4 +-
.../accumulo/test/randomwalk/image/Verify.java | 5 ++-
.../accumulo/test/randomwalk/image/Write.java | 9 ++--
.../test/randomwalk/multitable/Write.java | 5 ++-
.../randomwalk/security/SecurityHelper.java | 7 ++--
.../test/randomwalk/security/TableOp.java | 9 ++--
.../test/randomwalk/sequential/Write.java | 5 ++-
.../test/randomwalk/shard/BulkInsert.java | 5 ++-
.../test/randomwalk/shard/ExportIndex.java | 5 ++-
.../accumulo/test/scalability/Ingest.java | 5 ++-
.../apache/accumulo/test/AuditMessageIT.java | 6 +--
.../org/apache/accumulo/test/NamespacesIT.java | 8 ++--
.../java/org/apache/accumulo/test/VolumeIT.java | 6 +--
.../accumulo/test/functional/AddSplitIT.java | 5 ++-
.../test/functional/BadIteratorMincIT.java | 6 +--
.../test/functional/BatchScanSplitIT.java | 7 ++--
.../test/functional/BatchWriterFlushIT.java | 5 ++-
.../accumulo/test/functional/BulkFileIT.java | 5 ++-
.../accumulo/test/functional/ConcurrencyIT.java | 7 ++--
.../accumulo/test/functional/ConstraintIT.java | 23 ++++++-----
.../test/functional/CreateAndUseIT.java | 5 ++-
.../test/functional/DeleteEverythingIT.java | 4 +-
.../test/functional/DeleteRowsSplitIT.java | 6 +--
.../accumulo/test/functional/LargeRowIT.java | 7 ++--
.../accumulo/test/functional/NativeMapIT.java | 15 +++----
.../test/functional/ScanIteratorIT.java | 7 ++--
.../accumulo/test/functional/ScanRangeIT.java | 5 ++-
.../test/functional/ScanSessionTimeOutIT.java | 5 ++-
.../test/functional/SplitRecoveryIT.java | 3 +-
.../accumulo/test/functional/TabletIT.java | 4 +-
.../accumulo/test/functional/VisibilityIT.java | 11 ++---
.../instrument/receivers/ZooSpanClient.java | 4 +-
215 files changed, 900 insertions(+), 667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/assemble/bin/bootstrap_config.sh
----------------------------------------------------------------------
diff --cc assemble/bin/bootstrap_config.sh
index fb643da,0000000..fd262f0
mode 100755,000000..100755
--- a/assemble/bin/bootstrap_config.sh
+++ b/assemble/bin/bootstrap_config.sh
@@@ -1,323 -1,0 +1,323 @@@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function usage {
+ cat <<EOF
+Usage: bootstrap_config.sh [-options]
+where options include (long options not available on all platforms):
+ -d, --dir Alternate directory to setup config files
+ -s, --size Supported sizes: '1GB' '2GB' '3GB' '512MB'
+ -n, --native Configure to use native libraries
+ -j, --jvm Configure to use the jvm
+ -o, --overwrite Overwrite the default config directory
+ -v, --version Specify the Apache Hadoop version supported versions: '1' '2'
+ -h, --help Print this help message
+EOF
+}
+
+# Start: Resolve Script Directory
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+# Stop: Resolve Script Directory
+
+#
+# Resolve accumulo home for bootstrapping
+#
+ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd )
+TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates"
+CONF_DIR="${ACCUMULO_HOME}/conf"
+ACCUMULO_SITE=accumulo-site.xml
+ACCUMULO_ENV=accumulo-env.sh
+
+SIZE=
+TYPE=
+HADOOP_VERSION=
+OVERWRITE="0"
+BASE_DIR=
+
+#Execute getopt
+if [[ $(uname -s) == "Linux" ]]; then
+ args=$(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@")
+else # Darwin, BSD
+ args=$(getopt b:d:s:njov:h $*)
+fi
+
+#Bad arguments
+if [[ $? != 0 ]]; then
+ usage 1>&2
+ exit 1
+fi
+eval set -- $args
+
+for i
+do
+ case "$i" in
+ -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers
+ BASE_DIR=$2; shift
+ shift;;
+ -d|--dir)
+ CONF_DIR=$2; shift
+ shift;;
+ -s|--size)
+ SIZE=$2; shift
+ shift;;
+ -n|--native)
+ TYPE=native
+ shift;;
+ -j|--jvm)
+ TYPE=jvm
+ shift;;
+ -o|--overwrite)
+ OVERWRITE=1
+ shift;;
+ -v|--version)
+ HADOOP_VERSION=$2; shift
+ shift;;
+ -h|--help)
+ usage
+ exit 0
+ shift;;
+ --)
+ shift
+ break;;
+ esac
+done
+
+while [[ "${OVERWRITE}" = "0" ]]; do
+ if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then
+ echo "Warning your current config files in ${CONF_DIR} will be overwritten!"
+ echo
+ echo "How would you like to proceed?:"
+ select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do
+ if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then
+ echo -n "Please specifiy new conf directory: "
+ read CONF_DIR
+ elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then
+ OVERWRITE=1
+ fi
+ break
+ done
+ else
+ OVERWRITE=1
+ fi
+done
- echo "Coppying configuration files to: ${CONF_DIR}"
++echo "Copying configuration files to: ${CONF_DIR}"
+
+#Native 1GB
+native_1GB_tServer="-Xmx128m -Xms128m"
+_1GB_master="-Xmx128m -Xms128m"
+_1GB_monitor="-Xmx64m -Xms64m"
+_1GB_gc="-Xmx64m -Xms64m"
+_1GB_other="-Xmx128m -Xms64m"
+
+_1GB_memoryMapMax="256M"
+native_1GB_nativeEnabled="true"
+_1GB_cacheDataSize="15M"
+_1GB_cacheIndexSize="40M"
+_1GB_sortBufferSize="50M"
+_1GB_waLogMaxSize="256M"
+
+#Native 2GB
+native_2GB_tServer="-Xmx256m -Xms256m"
+_2GB_master="-Xmx256m -Xms256m"
+_2GB_monitor="-Xmx128m -Xms64m"
+_2GB_gc="-Xmx128m -Xms128m"
+_2GB_other="-Xmx256m -Xms64m"
+
+_2GB_memoryMapMax="512M"
+native_2GB_nativeEnabled="true"
+_2GB_cacheDataSize="30M"
+_2GB_cacheIndexSize="80M"
+_2GB_sortBufferSize="50M"
+_2GB_waLogMaxSize="512M"
+
+#Native 3GB
+native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m"
+_3GB_master="-Xmx1g -Xms1g"
+_3GB_monitor="-Xmx1g -Xms256m"
+_3GB_gc="-Xmx256m -Xms256m"
+_3GB_other="-Xmx1g -Xms256m"
+
+_3GB_memoryMapMax="1G"
+native_3GB_nativeEnabled="true"
+_3GB_cacheDataSize="128M"
+_3GB_cacheIndexSize="128M"
+_3GB_sortBufferSize="200M"
+_3GB_waLogMaxSize="1G"
+
+#Native 512MB
+native_512MB_tServer="-Xmx48m -Xms48m"
+_512MB_master="-Xmx128m -Xms128m"
+_512MB_monitor="-Xmx64m -Xms64m"
+_512MB_gc="-Xmx64m -Xms64m"
+_512MB_other="-Xmx128m -Xms64m"
+
+_512MB_memoryMapMax="80M"
+native_512MB_nativeEnabled="true"
+_512MB_cacheDataSize="7M"
+_512MB_cacheIndexSize="20M"
+_512MB_sortBufferSize="50M"
+_512MB_waLogMaxSize="100M"
+
+#JVM 1GB
+jvm_1GB_tServer="-Xmx384m -Xms384m"
+
+jvm_1GB_nativeEnabled="false"
+
+#JVM 2GB
+jvm_2GB_tServer="-Xmx768m -Xms768m"
+
+jvm_2GB_nativeEnabled="false"
+
+#JVM 3GB
+jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G"
+
+jvm_3GB_nativeEnabled="false"
+
+#JVM 512MB
+jvm_512MB_tServer="-Xmx128m -Xms128m"
+
+jvm_512MB_nativeEnabled="false"
+
+
+if [[ -z "${SIZE}" ]]; then
+ echo "Choose the heap configuration:"
+ select DIRNAME in 1GB 2GB 3GB 512MB; do
+ echo "Using '${DIRNAME}' configuration"
+ SIZE=${DIRNAME}
+ break
+ done
+elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB" && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then
+ echo "Invalid memory size"
+ echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'"
+ exit 1
+fi
+
+if [[ -z "${TYPE}" ]]; then
+ echo
+ echo "Choose the Accumulo memory-map type:"
+ select TYPENAME in Java Native; do
+ if [[ "${TYPENAME}" == "Native" ]]; then
+ TYPE="native"
+ echo "Don't forget to build the native libraries using the bin/build_native_library.sh script"
+ elif [[ "${TYPENAME}" == "Java" ]]; then
+ TYPE="jvm"
+ fi
+ echo "Using '${TYPE}' configuration"
+ echo
+ break
+ done
+fi
+
+if [[ -z "${HADOOP_VERSION}" ]]; then
+ echo
+ echo "Choose the Apache Hadoop version:"
+ select HADOOP in 'HADOOP 1' 'HADOOP 2' ; do
+ if [ "${HADOOP}" == "HADOOP 2" ]; then
+ HADOOP_VERSION="2"
+ elif [ "${HADOOP}" == "HADOOP 1" ]; then
+ HADOOP_VERSION="1"
+ fi
+ echo "Using Apache Hadoop version '${HADOOP_VERSION}' configuration"
+ echo
+ break
+ done
+elif [[ "${HADOOP_VERSION}" != "1" && "${HADOOP_VERSION}" != "2" ]]; then
+ echo "Invalid Apache Hadoop version"
+ echo "Supported Apache Hadoop versions: '1' '2'"
+ exit 1
+fi
+
+for var in SIZE TYPE HADOOP_VERSION; do
+ if [[ -z ${!var} ]]; then
+ echo "Invalid $var configuration"
+ exit 1
+ fi
+done
+
+TSERVER="${TYPE}_${SIZE}_tServer"
+MASTER="_${SIZE}_master"
+MONITOR="_${SIZE}_monitor"
+GC="_${SIZE}_gc"
+OTHER="_${SIZE}_other"
+
+MEMORY_MAP_MAX="_${SIZE}_memoryMapMax"
+NATIVE="${TYPE}_${SIZE}_nativeEnabled"
+CACHE_DATA_SIZE="_${SIZE}_cacheDataSize"
+CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize"
+SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize"
+WAL_MAX_SIZE="_${SIZE}_waLogMaxSize"
+
+MAVEN_PROJ_BASEDIR=""
+
+if [[ ! -z "${BASE_DIR}" ]]; then
+ MAVEN_PROJ_BASEDIR="\n <property>\n <name>general.maven.project.basedir</name>\n <value>${BASE_DIR}</value>\n </property>\n"
+fi
+
+#Configure accumulo-env.sh
+mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/
+sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \
+ -e "s/\${masterHigh_masterLow}/${!MASTER}/" \
+ -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \
+ -e "s/\${gcHigh_gcLow}/${!GC}/" \
+ -e "s/\${otherHigh_otherLow}/${!OTHER}/" \
+ ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV
+
+#Configure accumulo-site.xml
+sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \
+ -e "s/\${nativeEnabled}/${!NATIVE}/" \
+ -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \
+ -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \
+ -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \
+ -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \
+ -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE
+
+#Configure for hadoop 1
+if [[ "$HADOOP_VERSION" = "1" ]]; then
+ sed -e 's/^test -z \"$HADOOP_CONF_DIR\"/#test -z \"$HADOOP_CONF_DIR\"/' -e 's/^# test -z "$HADOOP_CONF_DIR"/test -z \"$HADOOP_CONF_DIR\"/' ${CONF_DIR}/$ACCUMULO_ENV > temp
+ mv temp ${CONF_DIR}/$ACCUMULO_ENV
+ sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements -->\n <!--/' \
+ -e 's/<!-- End Hadoop 2 requirements -->/-->\n <!-- End Hadoop 2 requirements -->/' \
+ ${CONF_DIR}/$ACCUMULO_SITE > temp
+ mv temp ${CONF_DIR}/$ACCUMULO_SITE
+fi
+
+#Additional setup steps for native configuration.
+if [[ "${TYPE}" = "native" ]]; then
+ if [[ "$(uname)" = 'Linux' ]]; then
+ if [[ -z $HADOOP_PREFIX ]]; then
+ echo "HADOOP_PREFIX not set cannot automatically configure LD_LIBRARY_PATH"
+ else
+ NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null)
+ if [[ -z $NATIVE_LIB ]]; then
+ echo -e "Native libraries could not be found for your sytem in: $HADOOP_PREFIX"
+ else
+ sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp
+ mv temp ${CONF_DIR}/$ACCUMULO_ENV
+ echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH"
+ fi
+ fi
+ fi
+ echo -e "Please remember to compile the native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed."
+fi
+echo "Setup complete"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 2a570f4,240de73..9897262
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -16,17 -16,37 +16,26 @@@
*/
package org.apache.accumulo.core;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.ColumnFQ;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
+ import com.google.common.base.Charsets;
+
public class Constants {
- public static final Charset UTF8 = Charset.forName("UTF-8");
+ /**
- * @deprecated since 1.5.3; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
++ * @deprecated since 1.5.3, 1.6.2; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
+ * {@link Charset#forName(String)} with "UTF-8" instead
+ */
+ @Deprecated
+ public static final Charset UTF8 = UTF_8;
public static final String VERSION = FilteredConstants.VERSION;
-
- // versions should never be negative
- public static final Integer WIRE_VERSION = 2;
- public static final int DATA_VERSION = 5;
- public static final int PREV_DATA_VERSION = 4;
-
+
// Zookeeper locations
public static final String ZROOT = "/accumulo";
public static final String ZINSTANCES = "/instances";
@@@ -56,62 -76,117 +65,62 @@@
public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
-
+
public static final String ZCONFIG = "/config";
-
+
public static final String ZTSERVERS = "/tservers";
-
+
public static final String ZDEAD = "/dead";
- public static final String ZDEADTSERVERS = "/dead/tservers";
-
+ public static final String ZDEADTSERVERS = ZDEAD + "/tservers";
+
public static final String ZTRACERS = "/tracers";
-
+
public static final String ZPROBLEMS = "/problems";
- public static final String ZUSERS = "/users";
-
+
public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
-
+
public static final String ZFATE = "/fate";
-
+
public static final String ZNEXT_FILE = "/next_file";
-
+
public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
-
+
public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
public static final String ZRECOVERY = "/recovery";
-
- public static final String METADATA_TABLE_ID = "!0";
- public static final String METADATA_TABLE_NAME = "!METADATA";
- public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
- public static final String TABLE_TABLET_LOCATION = "/table_info";
- public static final String ZTABLE_LOCKS = "/table_locks";
-
- // reserved keyspace is any row that begins with a tilde '~' character
- public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
- public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
- public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
- public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
- public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
- public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
- public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
- public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
- public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
-
- public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
- public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
- public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
- public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
- public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
- public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
- public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
-
- // README : very important that prevRow sort last to avoid race conditions between
- // garbage collector and split
- public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
- // else for that tablet
- public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
- public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
- public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
- public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
- public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
- public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
- public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
-
- public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
- public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
- public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
- public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
- public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
-
- public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
- new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
- public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
-
- public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
- null);
- public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
- METADATA_TABLE_ID), null), true);
-
+
/**
- * @deprecated since 1.5.3; not used, but specifies UTF-8
+ * Initial tablet directory name for the default tablet in all tables
*/
- @Deprecated
- public static final String VALUE_ENCODING = UTF_8.name();
-
+ public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
+
+ public static final String ZTABLE_LOCKS = "/table_locks";
+
public static final String BULK_PREFIX = "b-";
- public static final String OLD_BULK_PREFIX = "bulk_";
-
- // note: all times are in milliseconds
-
- public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
-
- public static final long MIN_MASTER_LOOP_TIME = 1000;
- public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
- public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
-
+
+ public static final String CLONE_PREFIX = "c-";
- public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF8);
++ public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF_8);
+
+ // this affects the table client caching of metadata
+ public static final int SCAN_BATCH_SIZE = 1000;
+
+ // Scanners will default to fetching 3 batches of Key/Value pairs before asynchronously
+ // fetching the next batch.
+ public static final long SCANNER_DEFAULT_READAHEAD_THRESHOLD = 3l;
+
// Security configuration
public static final String PW_HASH_ALGORITHM = "SHA-256";
-
- // Representation of an empty set of authorizations
- // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
- public static final Authorizations NO_AUTHS = new Authorizations();
-
- public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
-
+
+ /**
+ * @deprecated since 1.6.0; Use {@link Authorizations#EMPTY} instead
+ */
+ @Deprecated
+ public static final Authorizations NO_AUTHS = Authorizations.EMPTY;
+
public static final int MAX_DATA_TO_PRINT = 64;
- public static final int CLIENT_RETRIES = 5;
- public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
- public static final String OLD_PACKAGE_NAME = "cloudbase";
- public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
public static final String MAPFILE_EXTENSION = "map";
public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
-
+
public static final String EXPORT_METADATA_FILE = "metadata.bin";
public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
public static final String EXPORT_FILE = "exportMetadata.zip";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
index 35a9067,5134521..041c9e6
--- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
@@@ -16,6 -16,9 +16,8 @@@
*/
package org.apache.accumulo.core.cli;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 6d35757,251d6a7..a9c3292
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -16,6 -16,11 +16,8 @@@
*/
package org.apache.accumulo.core.client;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
index 63e3497,0000000..6925f18
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
@@@ -1,210 -1,0 +1,212 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.client.admin.ActiveCompaction;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering the accumulo instance
+ */
+public class InstanceOperationsImpl implements InstanceOperations {
+ private Instance instance;
+ private Credentials credentials;
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the Credential, containing principal and Authentication Token
+ */
+ public InstanceOperationsImpl(Instance instance, Credentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ @Override
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property, value);
+ }
+ });
+ }
+
+ @Override
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property);
+ }
+ });
+ }
+
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.CURRENT);
+ }
+ });
+ }
+
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.SITE);
+ }
+ });
+ }
+
+ @Override
+ public List<String> getTabletServers() {
+ ZooCache cache = new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ List<String> results = new ArrayList<String>();
+ for (String candidate : cache.getChildren(path)) {
+ List<String> children = cache.getChildren(path + "/" + candidate);
+ if (children != null && children.size() > 0) {
+ List<String> copy = new ArrayList<String>(children);
+ Collections.sort(copy);
+ byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
- if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
++ if (data != null && !"master".equals(new String(data, UTF_8))) {
+ results.add(candidate);
+ }
+ }
+ }
+ return results;
+ }
+
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
+
+ List<ActiveScan> as = new ArrayList<ActiveScan>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials.toThrift(instance))) {
+ try {
+ as.add(new ActiveScanImpl(instance, activeScan));
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkClass(Tracer.traceInfo(), credentials.toThrift(instance), className, asTypeName);
+ }
+ });
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
+
+ List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(),
+ credentials.toThrift(instance))) {
+ as.add(new ActiveCompactionImpl(instance, activeCompaction));
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+ TTransport transport = null;
+ try {
+ transport = ThriftUtil.createTransport(AddressUtil.parseAddress(tserver, false), ServerConfigurationUtil.getConfiguration(instance));
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ client.getTabletServerStatus(Tracer.traceInfo(), credentials.toThrift(instance));
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloException(e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (transport != null) {
+ transport.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
index 30f0266,0000000..f32940c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
@@@ -1,154 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.security.SecurityPermission;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.util.ArgumentChecker.Validator;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+
+public class Namespaces {
+ public static final String VALID_NAME_REGEX = "^\\w*$";
+ public static final Validator<String> VALID_NAME = new Validator<String>() {
+ @Override
+ public boolean isValid(String namespace) {
+ return namespace != null && namespace.matches(VALID_NAME_REGEX);
+ }
+
+ @Override
+ public String invalidMessage(String namespace) {
+ if (namespace == null)
+ return "Namespace cannot be null";
+ return "Namespaces must only contain word characters (letters, digits, and underscores): " + namespace;
+ }
+ };
+
+ public static final Validator<String> NOT_DEFAULT = new Validator<String>() {
+ @Override
+ public boolean isValid(String namespace) {
+ return !Namespaces.DEFAULT_NAMESPACE.equals(namespace);
+ }
+
+ @Override
+ public String invalidMessage(String namespace) {
+ return "Namespace cannot be the reserved empty namespace";
+ }
+ };
+
+ public static final Validator<String> NOT_ACCUMULO = new Validator<String>() {
+ @Override
+ public boolean isValid(String namespace) {
+ return !Namespaces.ACCUMULO_NAMESPACE.equals(namespace);
+ }
+
+ @Override
+ public String invalidMessage(String namespace) {
+ return "Namespace cannot be the reserved namespace, " + Namespaces.ACCUMULO_NAMESPACE;
+ }
+ };
+
+ private static SecurityPermission TABLES_PERMISSION = new SecurityPermission("tablesPermission");
+
+ public static final String DEFAULT_NAMESPACE_ID = "+default";
+ public static final String DEFAULT_NAMESPACE = "";
+ public static final String ACCUMULO_NAMESPACE_ID = "+accumulo";
+ public static final String ACCUMULO_NAMESPACE = "accumulo";
+
+ private static ZooCache getZooCache(Instance instance) {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TABLES_PERMISSION);
+ }
+ return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ }
+
+ private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
+ ZooCache zc = getZooCache(instance);
+
+ List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
+
+ TreeMap<String,String> namespaceMap = new TreeMap<String,String>();
+
+ for (String id : namespaceIds) {
+ byte[] path = zc.get(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + id + Constants.ZNAMESPACE_NAME);
+ if (path != null) {
+ if (nameAsKey)
- namespaceMap.put(new String(path, Constants.UTF8), id);
++ namespaceMap.put(new String(path, UTF_8), id);
+ else
- namespaceMap.put(id, new String(path, Constants.UTF8));
++ namespaceMap.put(id, new String(path, UTF_8));
+ }
+ }
+ return namespaceMap;
+ }
+
+ public static boolean exists(Instance instance, String namespaceId) {
+ ZooCache zc = getZooCache(instance);
+ List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
+ return namespaceIds.contains(namespaceId);
+ }
+
+ public static String getNamespaceId(Instance instance, String namespace) throws NamespaceNotFoundException {
+ String id = getNameToIdMap(instance).get(namespace);
+ if (id == null)
+ throw new NamespaceNotFoundException(null, namespace, "getNamespaceId() failed to find namespace");
+ return id;
+ }
+
+ public static String getNamespaceName(Instance instance, String namespaceId) throws NamespaceNotFoundException {
+ String namespaceName = getIdToNameMap(instance).get(namespaceId);
+ if (namespaceName == null)
+ throw new NamespaceNotFoundException(namespaceId, null, "getNamespaceName() failed to find namespace");
+ return namespaceName;
+ }
+
+ public static SortedMap<String,String> getNameToIdMap(Instance instance) {
+ return getMap(instance, true);
+ }
+
+ public static SortedMap<String,String> getIdToNameMap(Instance instance) {
+ return getMap(instance, false);
+ }
+
+ public static List<String> getTableIds(Instance instance, String namespaceId) throws NamespaceNotFoundException {
+ String namespace = getNamespaceName(instance, namespaceId);
+ List<String> names = new LinkedList<String>();
+ for (Entry<String,String> nameToId : Tables.getNameToIdMap(instance).entrySet())
+ if (namespace.equals(Tables.qualify(nameToId.getKey()).getFirst()))
+ names.add(nameToId.getValue());
+ return names;
+ }
+
+ public static List<String> getTableNames(Instance instance, String namespaceId) throws NamespaceNotFoundException {
+ String namespace = getNamespaceName(instance, namespaceId);
+ List<String> names = new LinkedList<String>();
+ for (String name : Tables.getNameToIdMap(instance).keySet())
+ if (namespace.equals(Tables.qualify(name).getFirst()))
+ names.add(name);
+ return names;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
index 8da530b,1fcebe7..70bdbb0
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
@@@ -16,7 -16,11 +16,9 @@@
*/
package org.apache.accumulo.core.client.impl;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
@@@ -128,10 -142,13 +130,10 @@@ public class ServerClient
for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
byte[] data = ZooUtil.getLockData(zc, path);
- if (data != null && !new String(data, Constants.UTF8).equals("master"))
- if (data != null) {
- String tserverData = new String(data, UTF_8);
- if (!tserverData.equals("master")) {
- servers.add(new ThriftTransportKey(new ServerServices(tserverData).getAddressString(Service.TSERV_CLIENT), conf.getPort(Property.TSERV_CLIENTPORT),
- rpcTimeout));
- }
- }
++ if (data != null && !new String(data, UTF_8).equals("master"))
+ servers.add(new ThriftTransportKey(
+ new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT),
+ rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance))));
}
boolean opened = false;
[39/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index bfd4818,0000000..e6e31eb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -1,238 -1,0 +1,239 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Arrays;
+import java.util.HashSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Conditions that must be met on a particular column in a row.
+ *
+ * @since 1.6.0
+ */
+public class Condition {
+
+ private ByteSequence cf;
+ private ByteSequence cq;
+ private ByteSequence cv;
+ private ByteSequence val;
+ private Long ts;
+ private IteratorSetting iterators[] = new IteratorSetting[0];
+ private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+
+
+ public Condition(CharSequence cf, CharSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
- this.cf = new ArrayByteSequence(cf.toString().getBytes(Constants.UTF8));
- this.cq = new ArrayByteSequence(cq.toString().getBytes(Constants.UTF8));
++ this.cf = new ArrayByteSequence(cf.toString().getBytes(UTF_8));
++ this.cq = new ArrayByteSequence(cq.toString().getBytes(UTF_8));
+ this.cv = EMPTY;
+ }
+
+ public Condition(byte[] cf, byte[] cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf);
+ this.cq = new ArrayByteSequence(cq);
+ this.cv = EMPTY;
+ }
+
+ public Condition(Text cf, Text cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf.getBytes(), 0, cf.getLength());
+ this.cq = new ArrayByteSequence(cq.getBytes(), 0, cq.getLength());
+ this.cv = EMPTY;
+ }
+
+ public Condition(ByteSequence cf, ByteSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = cf;
+ this.cq = cq;
+ this.cv = EMPTY;
+ }
+
+ public ByteSequence getFamily() {
+ return cf;
+ }
+
+ public ByteSequence getQualifier() {
+ return cq;
+ }
+
+ /**
+ * Sets the version for the column to check. If this is not set then the latest column will be checked, unless iterators do something different.
+ *
+ * @return returns this
+ */
+
+ public Condition setTimestamp(long ts) {
+ this.ts = ts;
+ return this;
+ }
+
+ public Long getTimestamp() {
+ return ts;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(CharSequence value) {
+ ArgumentChecker.notNull(value);
- this.val = new ArrayByteSequence(value.toString().getBytes(Constants.UTF8));
++ this.val = new ArrayByteSequence(value.toString().getBytes(UTF_8));
+ return this;
+ }
+
+ /**
+ * This method sets the expected value of a column. Inorder for the condition to pass the column must exist and have this value. If a value is not set, then
+ * the column must be absent for the condition to pass.
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(byte[] value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value);
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(Text value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value.getBytes(), 0, value.getLength());
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(ByteSequence value) {
+ ArgumentChecker.notNull(value);
+ this.val = value;
+ return this;
+ }
+
+ public ByteSequence getValue() {
+ return val;
+ }
+
+ /**
+ * Sets the visibility for the column to check. If not set it defaults to empty visibility.
+ *
+ * @return returns this
+ */
+
+ public Condition setVisibility(ColumnVisibility cv) {
+ ArgumentChecker.notNull(cv);
+ this.cv = new ArrayByteSequence(cv.getExpression());
+ return this;
+ }
+
+ public ByteSequence getVisibility() {
+ return cv;
+ }
+
+ /**
+ * Set iterators to use when reading the columns value. These iterators will be applied in addition to the iterators configured for the table. Using iterators
+ * its possible to test other conditions, besides equality and absence, like less than. On the server side the iterators will be seeked using a range that
+ * covers only the family, qualifier, and visibility (if the timestamp is set then it will be used to narrow the range). Value equality will be tested using
+ * the first entry returned by the iterator stack.
+ *
+ * @return returns this
+ */
+
+ public Condition setIterators(IteratorSetting... iterators) {
+ ArgumentChecker.notNull(iterators);
+
+ if (iterators.length > 1) {
+ HashSet<String> names = new HashSet<String>();
+ HashSet<Integer> prios = new HashSet<Integer>();
+
+ for (IteratorSetting iteratorSetting : iterators) {
+ if (!names.add(iteratorSetting.getName()))
+ throw new IllegalArgumentException("iterator name used more than once " + iteratorSetting.getName());
+ if (!prios.add(iteratorSetting.getPriority()))
+ throw new IllegalArgumentException("iterator priority used more than once " + iteratorSetting.getPriority());
+ }
+ }
+
+ this.iterators = iterators;
+ return this;
+ }
+
+ public IteratorSetting[] getIterators() {
+ return iterators;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o == null || !(o instanceof Condition)) {
+ return false;
+ }
+ Condition c = (Condition) o;
+ if (!(c.cf.equals(cf))) {
+ return false;
+ }
+ if (!(c.cq.equals(cq))) {
+ return false;
+ }
+ if (!(c.cv.equals(cv))) {
+ return false;
+ }
+ if (!(c.val == null ? val == null : c.val.equals(val))) {
+ return false;
+ }
+ if (!(c.ts == null ? ts == null : c.ts.equals(ts))) {
+ return false;
+ }
+ if (!(Arrays.equals(c.iterators, iterators))) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 31 * result + cf.hashCode();
+ result = 31 * result + cq.hashCode();
+ result = 31 * result + cv.hashCode();
+ result = 31 * result + (val == null ? 0 : val.hashCode());
+ result = 31 * result + (ts == null ? 0 : ts.hashCode());
+ result = 31 * result + Arrays.hashCode(iterators);
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Value.java
index 11e60e1,bc1f71d..d256107
--- a/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@@ -25,9 -26,6 +26,8 @@@ import java.io.IOException
import java.nio.ByteBuffer;
import java.util.List;
+import com.google.common.base.Preconditions;
+
- import org.apache.accumulo.core.Constants;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
index 8006114,0b220ee..7073d11
--- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
@@@ -24,10 -26,7 +26,9 @@@ import java.util.Collections
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.util.ArgumentChecker;
@@@ -195,13 -166,12 +196,13 @@@ public class Authorizations implements
}
/**
- * Retrieve a serialized form of the underlying set of authorizations.
+ * Returns a serialized form of these authorizations.
*
- * @see #Authorizations(byte[])
+ * @return serialized form of these authorizations, as a string encoded in UTF-8
+ * @see #serialize()
*/
public byte[] getAuthorizationsArray() {
- return serialize().getBytes(Constants.UTF8);
+ return serialize().getBytes(UTF_8);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index e76b0ef,66b68ab..e8f4e97
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -243,9 -223,9 +244,9 @@@ public class ColumnVisibility
/*
* Walks an expression's AST and appends a string representation to a supplied StringBuilder. This method adds parens where necessary.
*/
- private static void stringify(Node root, byte[] expression, StringBuilder out) {
+ public static void stringify(Node root, byte[] expression, StringBuilder out) {
if (root.type == NodeType.TERM) {
- out.append(new String(expression, root.start, root.end - root.start, Constants.UTF8));
+ out.append(new String(expression, root.start, root.end - root.start, UTF_8));
} else {
String sep = "";
for (Node c : root.children) {
@@@ -314,12 -293,12 +315,12 @@@
while (index < expression.length) {
switch (expression[index++]) {
case '&': {
- expr = processTerm(termStart, index - 1, expr, expression);
+ expr = processTerm(subtermStart, index - 1, expr, expression);
if (result != null) {
if (!result.type.equals(NodeType.AND))
- throw new BadArgumentException("cannot mix & and |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("cannot mix & and |", new String(expression, UTF_8), index - 1);
} else {
- result = new Node(NodeType.AND);
+ result = new Node(NodeType.AND, wholeTermStart);
}
result.add(expr);
expr = null;
@@@ -328,12 -307,12 +329,12 @@@
break;
}
case '|': {
- expr = processTerm(termStart, index - 1, expr, expression);
+ expr = processTerm(subtermStart, index - 1, expr, expression);
if (result != null) {
if (!result.type.equals(NodeType.OR))
- throw new BadArgumentException("cannot mix | and &", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("cannot mix | and &", new String(expression, UTF_8), index - 1);
} else {
- result = new Node(NodeType.OR);
+ result = new Node(NodeType.OR, wholeTermStart);
}
result.add(expr);
expr = null;
@@@ -343,18 -322,18 +344,18 @@@
}
case '(': {
parens++;
- if (termStart != index - 1 || expr != null)
+ if (subtermStart != index - 1 || expr != null)
- throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
expr = parse_(expression);
- termStart = index;
- termComplete = false;
+ subtermStart = index;
+ subtermComplete = false;
break;
}
case ')': {
parens--;
- Node child = processTerm(termStart, index - 1, expr, expression);
+ Node child = processTerm(subtermStart, index - 1, expr, expression);
if (child == null && result == null)
- throw new BadArgumentException("empty expression not allowed", new String(expression, Constants.UTF8), index);
+ throw new BadArgumentException("empty expression not allowed", new String(expression, UTF_8), index);
if (result == null)
return child;
if (result.type == child.type)
@@@ -366,8 -345,8 +367,8 @@@
return result;
}
case '"': {
- if (termStart != index - 1)
+ if (subtermStart != index - 1)
- throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
while (index < expression.length && expression[index] != '"') {
if (expression[index] == '\\') {
@@@ -379,20 -358,20 +380,20 @@@
}
if (index == expression.length)
- throw new BadArgumentException("unclosed quote", new String(expression, Constants.UTF8), subtermStart);
- throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), termStart);
-
- if (termStart + 1 == index)
- throw new BadArgumentException("empty term", new String(expression, UTF_8), termStart);
++ throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), subtermStart);
+ if (subtermStart + 1 == index)
- throw new BadArgumentException("empty term", new String(expression, Constants.UTF8), subtermStart);
++ throw new BadArgumentException("empty term", new String(expression, UTF_8), subtermStart);
+
index++;
- termComplete = true;
+ subtermComplete = true;
break;
}
default: {
- if (termComplete)
+ if (subtermComplete)
- throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
byte c = expression[index - 1];
if (!Authorizations.isValidAuthChar(c))
@@@ -514,12 -482,10 +515,12 @@@
* .
* ColumnVisibility cv = new ColumnVisibility(quote("A#C") + "&" + quote("FOO"));
* </pre>
- *
+ *
+ * @param term term to quote
+ * @return quoted term (unquoted if unnecessary)
*/
public static String quote(String term) {
- return new String(quote(term.getBytes(Constants.UTF8)), Constants.UTF8);
+ return new String(quote(term.getBytes(UTF_8)), UTF_8);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Credentials.java
index 582b4e0,31fe18d..0ecc31d
--- a/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
@@@ -16,17 -16,10 +16,18 @@@
*/
package org.apache.accumulo.core.security;
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.nio.ByteBuffer;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Base64;
/**
* A wrapper for internal use. This class carries the instance, principal, and authentication token for use in the public API, in a non-serialized form. This is
@@@ -53,102 -36,9 +54,102 @@@ public class Credentials
this.principal = principal;
this.token = token;
}
-
- public TCredentials toThrift() {
- return CredentialHelper.createSquelchError(principal, token, instance.getInstanceID());
+
+ /**
+ * Gets the principal.
+ *
+ * @return unique identifier for the entity (e.g. a user or service) authorized for these credentials
+ */
+ public String getPrincipal() {
+ return principal;
+ }
+
+ /**
+ * Gets the authentication token.
+ *
+ * @return authentication token used to prove that the principal for these credentials has been properly verified
+ */
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ /**
+ * Converts the current object to the relevant thrift type. The object returned from this contains a non-destroyable version of the
+ * {@link AuthenticationToken}, so this should be used just before placing on the wire, and references to it should be tightly controlled.
+ *
+ * @param instance
+ * client instance
+ * @return Thrift credentials
+ * @throws RuntimeException
+ * if the authentication token has been destroyed (expired)
+ */
+ public TCredentials toThrift(Instance instance) {
+ TCredentials tCreds = new TCredentials(getPrincipal(), getToken().getClass().getName(),
+ ByteBuffer.wrap(AuthenticationTokenSerializer.serialize(getToken())), instance.getInstanceID());
+ if (getToken().isDestroyed())
+ throw new RuntimeException("Token has been destroyed", new AccumuloSecurityException(getPrincipal(), SecurityErrorCode.TOKEN_EXPIRED));
+ return tCreds;
+ }
+
+ /**
+ * Converts a given thrift object to our internal Credentials representation.
+ * @param serialized a Thrift encoded set of credentials
+ * @return a new Credentials instance; destroy the token when you're done.
+ */
+ public static Credentials fromThrift(TCredentials serialized) {
+ return new Credentials(serialized.getPrincipal(), AuthenticationTokenSerializer.deserialize(serialized.getTokenClassName(), serialized.getToken()));
+ }
+
+ /**
+ * Converts the current object to a serialized form. The object returned from this contains a non-destroyable version of the {@link AuthenticationToken}, so
+ * references to it should be tightly controlled.
+ *
+ * @return serialized form of these credentials
+ */
+ public final String serialize() {
- return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(Constants.UTF8))) + ":"
- + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(Constants.UTF8))) + ":"
++ return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(UTF_8))) + ":"
++ + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(UTF_8))) + ":"
+ + (getToken() == null ? "-" : Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(getToken())));
+ }
+
+ /**
+ * Converts the serialized form to an instance of {@link Credentials}. The original serialized form will not be affected.
+ *
+ * @param serializedForm
+ * serialized form of credentials
+ * @return deserialized credentials
+ */
+ public static final Credentials deserialize(String serializedForm) {
+ String[] split = serializedForm.split(":", 3);
- String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), Constants.UTF8);
- String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), Constants.UTF8);
++ String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), UTF_8);
++ String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), UTF_8);
+ AuthenticationToken token = null;
+ if (!split[2].equals("-")) {
+ byte[] tokenBytes = Base64.decodeBase64(split[2]);
+ token = AuthenticationTokenSerializer.deserialize(tokenType, tokenBytes);
+ }
+ return new Credentials(principal, token);
+ }
+
+ @Override
+ public int hashCode() {
+ return getPrincipal() == null ? 0 : getPrincipal().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof Credentials))
+ return false;
+ Credentials other = Credentials.class.cast(obj);
+ boolean pEq = getPrincipal() == null ? (other.getPrincipal() == null) : (getPrincipal().equals(other.getPrincipal()));
+ if (!pEq)
+ return false;
+ boolean tEq = getToken() == null ? (other.getToken() == null) : (getToken().equals(other.getToken()));
+ return tEq;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getName() + ":" + getPrincipal() + ":" + (getToken() == null ? null : getToken().getClass().getName()) + ":<hidden>";
}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
index be5f008,9c5782b..178de91
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
@@@ -16,27 -16,17 +16,27 @@@
*/
package org.apache.accumulo.core.security;
- import java.text.ParseException;
+ import static com.google.common.base.Charsets.UTF_8;
- import org.apache.accumulo.core.Constants;
+ import java.text.ParseException;
+/**
+ * An exception thrown when a visibility string cannot be parsed.
+ */
public class VisibilityParseException extends ParseException {
private static final long serialVersionUID = 1L;
private String visibility;
-
+
+ /**
+ * Creates a new exception.
+ *
+ * @param reason reason string
+ * @param visibility visibility that could not be parsed
+ * @param errorOffset offset into visibility where parsing failed
+ */
public VisibilityParseException(String reason, byte[] visibility, int errorOffset) {
super(reason, errorOffset);
- this.visibility = new String(visibility, Constants.UTF8);
+ this.visibility = new String(visibility, UTF_8);
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
index 7a23c37,f6a6a01..8771832
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
@@@ -22,8 -24,6 +24,7 @@@ import java.util.Arrays
import java.util.Collection;
import java.util.List;
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.ByteSequence;
import org.apache.hadoop.io.Text;
public class ByteBufferUtil {
@@@ -62,18 -62,6 +63,18 @@@
}
public static String toString(ByteBuffer bytes) {
- return new String(bytes.array(), bytes.position(), bytes.remaining(), Constants.UTF8);
+ return new String(bytes.array(), bytes.position(), bytes.remaining(), UTF_8);
}
+
+ public static ByteBuffer toByteBuffers(ByteSequence bs) {
+ if (bs == null)
+ return null;
+
+ if (bs.isBackedByArray()) {
+ return ByteBuffer.wrap(bs.getBackingArray(), bs.offset(), bs.length());
+ } else {
+ // TODO create more efficient impl
+ return ByteBuffer.wrap(bs.toArray());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index 9f86db3,0000000..26a8c73
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@@ -1,117 -1,0 +1,118 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import jline.console.ConsoleReader;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOpts.Password;
+import org.apache.accumulo.core.cli.ClientOpts.PasswordConverter;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.Base64;
+
+import com.beust.jcommander.Parameter;
+
+public class CreateToken {
+
+ private static ConsoleReader reader = null;
+
+ private static ConsoleReader getConsoleReader() throws IOException {
+ if (reader == null)
+ reader = new ConsoleReader();
+ return reader;
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-u", "--user"}, description = "Connection user")
+ public String principal = null;
+
+ @Parameter(names = "-p", converter = PasswordConverter.class, description = "Connection password")
+ public Password password = null;
+
+ @Parameter(names = "--password", converter = PasswordConverter.class, description = "Enter the connection password", password = true)
+ public Password securePassword = null;
+
+ @Parameter(names = {"-tc", "--tokenClass"}, description = "The class of the authentication token")
+ public String tokenClassName = PasswordToken.class.getName();
+
+ @Parameter(names = {"-f", "--file"}, description = "The filename to save the auth token to. Multiple tokens can be stored in the same file,"
+ + " but only the first for each user will be recognized.")
+ public String tokenFile = null;
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(CreateToken.class.getName(), args);
+
+ Password pass = opts.password;
+ if (pass == null && opts.securePassword != null) {
+ pass = opts.securePassword;
+ }
+
+ try {
+ String principal = opts.principal;
+ if (principal == null) {
+ principal = getConsoleReader().readLine("Username (aka principal): ");
+ }
+
+ AuthenticationToken token = Class.forName(opts.tokenClassName).asSubclass(AuthenticationToken.class).newInstance();
+ Properties props = new Properties();
+ for (TokenProperty tp : token.getProperties()) {
+ String input;
+ if (pass != null && tp.getKey().equals("password")) {
+ input = pass.toString();
+ } else {
+ if (tp.getMask()) {
+ input = getConsoleReader().readLine(tp.getDescription() + ": ", '*');
+ } else {
+ input = getConsoleReader().readLine(tp.getDescription() + ": ");
+ }
+ }
+ props.put(tp.getKey(), input);
+ token.init(props);
+ }
+ String tokenBase64 = Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token));
+
+ String tokenFile = opts.tokenFile;
+ if (tokenFile == null) {
+ tokenFile = getConsoleReader().readLine("File to save auth token to: ");
+ }
+ File tf = new File(tokenFile);
+ if (!tf.exists()) {
+ tf.createNewFile();
+ }
- PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, Constants.UTF8.name());
++ PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, UTF_8.name());
+ String outString = principal + ":" + opts.tokenClassName + ":" + tokenBase64;
+ out.println(outString);
+ out.close();
+ System.out.println("Token written to " + tokenFile + ". Remember to upload it to hdfs.");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Encoding.java
index aff8f62,9999c3c..b26b363
--- a/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.util;
- import org.apache.accumulo.core.Constants;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import org.apache.commons.codec.binary.Base64;
+import org.apache.accumulo.core.util.Base64;
import org.apache.hadoop.io.Text;
public class Encoding {
@@@ -36,8 -38,10 +37,8 @@@
public static byte[] decodeBase64FileName(String node) {
while (node.length() % 4 != 0)
node += "=";
-
- node = node.replace('_', '/').replace('-', '+');
-
+ /* decode transparently handles URLSafe encodings */
- return Base64.decodeBase64(node.getBytes(Constants.UTF8));
+ return Base64.decodeBase64(node.getBytes(UTF_8));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java
index 02c8be3,3830108..05067b5
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@@ -21,7 -23,7 +23,6 @@@ import java.util.Iterator
import java.util.List;
import java.util.Map.Entry;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@@ -236,8 -233,8 +237,8 @@@ public class Merge
while (iterator.hasNext()) {
Entry<Key,Value> entry = iterator.next();
Key key = entry.getKey();
- if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- String[] sizeEntries = new String(entry.getValue().get(), Constants.UTF8).split(",");
+ String[] sizeEntries = new String(entry.getValue().get(), UTF_8).split(",");
if (sizeEntries.length == 2) {
tabletSize += Long.parseLong(sizeEntries[0]);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
index 1ebbf13,0000000..e71a377
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
@@@ -1,31 -1,0 +1,33 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.zookeeper.KeeperException;
+
+public class MonitorUtil {
+ public static String getLocation(Instance instance) throws KeeperException, InterruptedException {
+ ZooReader zr = new ZooReader(instance.getZooKeepers(), 5000);
+ byte[] loc = zr.getData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, null);
- return loc==null ? null : new String(loc, Constants.UTF8);
++ return loc==null ? null : new String(loc, UTF_8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index da5648c,755bee2..b187a76
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -557,14 -437,11 +559,14 @@@ public class Shell extends ShellOption
} catch (IOException e) {
log.warn("Unable to load history file at " + historyPath);
}
-
+
+ // Turn Ctrl+C into Exception instead of JVM exit
+ reader.setHandleUserInterrupt(true);
+
ShellCompletor userCompletor = null;
-
+
if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
- java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++ java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
try {
while (scanner.hasNextLine() && !hasExited()) {
execCommand(scanner.nextLine(), true, isVerbose());
@@@ -974,14 -805,14 +976,14 @@@
@Override
public void close() {}
};
-
+
public static class PrintFile implements PrintLine {
PrintWriter writer;
-
+
public PrintFile(String filename) throws FileNotFoundException {
- writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), Constants.UTF8)));
+ writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), UTF_8)));
}
-
+
@Override
public void print(String s) {
writer.println(s);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
index f06a639,c325612..90a3548
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
@@@ -19,9 -21,7 +21,8 @@@ import static com.google.common.base.Ch
import java.io.File;
import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.util.Base64;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
import org.apache.commons.cli.CommandLine;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
index bcb6c24,5c86ac9..44dce0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
@@@ -35,12 -36,12 +36,12 @@@ public class AuthenticateCommand extend
final String user = cl.getArgs()[0];
final String p = shellState.readMaskedLine("Enter current password for '" + user + "': ", '*');
if (p == null) {
- shellState.getReader().printNewline();
+ shellState.getReader().println();
return 0;
} // user canceled
- final byte[] password = p.getBytes(Constants.UTF8);
+ final byte[] password = p.getBytes(UTF_8);
final boolean valid = shellState.getConnector().securityOperations().authenticateUser(user, new PasswordToken(password));
- shellState.getReader().printString((valid ? "V" : "Not v") + "alid\n");
+ shellState.getReader().println((valid ? "V" : "Not v") + "alid");
return 0;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
index d2c73f7,8e39115..fc1af0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
@@@ -16,16 -16,16 +16,17 @@@
*/
package org.apache.accumulo.core.util.shell.commands;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.io.File;
import java.io.IOException;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableExistsException;
@@@ -73,12 -70,12 +74,12 @@@ public class CreateTableCommand extend
}
final SortedSet<Text> partitions = new TreeSet<Text>();
final boolean decode = cl.hasOption(base64Opt.getOpt());
-
+
if (cl.hasOption(createTableOptSplit.getOpt())) {
final String f = cl.getOptionValue(createTableOptSplit.getOpt());
-
+
String line;
- Scanner file = new Scanner(new File(f), Constants.UTF8.name());
+ Scanner file = new Scanner(new File(f), UTF_8.name());
try {
while (file.hasNextLine()) {
line = file.nextLine();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
index 61f60f8,4f1c793..06d36b9
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
@@@ -19,8 -21,6 +21,7 @@@ import static com.google.common.base.Ch
import java.security.SecureRandom;
import java.util.Random;
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.Base64;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
import org.apache.accumulo.core.util.shell.ShellCommandException;
@@@ -39,11 -40,12 +40,11 @@@ public class HiddenCommand extends Comm
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
if (rand.nextInt(10) == 0) {
shellState.getReader().beep();
- shellState.getReader().printNewline();
- shellState.getReader().printString(
+ shellState.getReader().println();
+ shellState.getReader().println(
new String(Base64.decodeBase64(("ICAgICAgIC4tLS4KICAgICAgLyAvXCBcCiAgICAgKCAvLS1cICkKICAgICAuPl8gIF88LgogICAgLyB8ICd8ICcgXAog"
+ "ICAvICB8Xy58Xy4gIFwKICAvIC98ICAgICAgfFwgXAogfCB8IHwgfFwvfCB8IHwgfAogfF98IHwgfCAgfCB8IHxffAogICAgIC8gIF9fICBcCiAgICAvICAv"
- + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(Constants.UTF8)), Constants.UTF8));
+ + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(UTF_8)), UTF_8));
- shellState.getReader().printNewline();
} else {
throw new ShellCommandException(ErrorCode.UNRECOGNIZED_COMMAND, getName());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/PasswdCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
index a0743f4,01fb92e..b8fc4c4
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
@@@ -41,10 -42,10 +42,10 @@@ public class UserCommand extends Comman
// of these methods fails
final String p = shellState.readMaskedLine("Enter password for user " + user + ": ", '*');
if (p == null) {
- shellState.getReader().printNewline();
+ shellState.getReader().println();
return 0;
} // user canceled
- pass = p.getBytes(Constants.UTF8);
+ pass = p.getBytes(UTF_8);
shellState.updateUser(user, new PasswordToken(pass));
return 0;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index 259cb06,3794f29..c276056
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@@ -16,6 -16,8 +16,7 @@@
*/
package org.apache.accumulo.core.client;
+ import static com.google.common.base.Charsets.UTF_8;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
index dde5575,0000000..dce742f
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
@@@ -1,157 -1,0 +1,158 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ZooKeeperInstanceTest {
+ private static final UUID IID = UUID.randomUUID();
+ private static final String IID_STRING = IID.toString();
+ private ClientConfiguration config;
+ private ZooCacheFactory zcf;
+ private ZooCache zc;
+ private ZooKeeperInstance zki;
+
+ private void mockIdConstruction(ClientConfiguration config) {
+ expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
+ expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
+ expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
+ }
+
+ private void mockNameConstruction(ClientConfiguration config) {
+ expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(null);
+ expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn("instance");
+ expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
+ }
+
+ @Before
+ public void setUp() {
+ config = createMock(ClientConfiguration.class);
+ mockNameConstruction(config);
+ replay(config);
+ zcf = createMock(ZooCacheFactory.class);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache("zk1", 30000)).andReturn(zc).anyTimes();
+ replay(zcf);
+ zki = new ZooKeeperInstance(config, zcf);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidConstruction() {
+ config = createMock(ClientConfiguration.class);
+ expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
+ mockNameConstruction(config);
+ replay(config);
+ new ZooKeeperInstance(config);
+ }
+
+ @Test
+ public void testSimpleGetters() {
+ assertEquals("instance", zki.getInstanceName());
+ assertEquals("zk1", zki.getZooKeepers());
+ assertEquals(30000, zki.getZooKeepersSessionTimeOut());
+ }
+
+ @Test
+ public void testGetInstanceID_FromCache() {
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
+ replay(zc);
+ assertEquals(IID_STRING, zki.getInstanceID());
+ }
+
+ @Test
+ public void testGetInstanceID_Direct() {
+ config = createMock(ClientConfiguration.class);
+ mockIdConstruction(config);
+ replay(config);
+ zki = new ZooKeeperInstance(config, zcf);
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
+ replay(zc);
+ assertEquals(IID_STRING, zki.getInstanceID());
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetInstanceID_NoMapping() {
+ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(null);
+ replay(zc);
+ zki.getInstanceID();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetInstanceID_IDMissingForName() {
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
+ replay(zc);
+ zki.getInstanceID();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetInstanceID_IDMissingForID() {
+ config = createMock(ClientConfiguration.class);
+ mockIdConstruction(config);
+ replay(config);
+ zki = new ZooKeeperInstance(config, zcf);
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
+ replay(zc);
+ zki.getInstanceID();
+ }
+
+ @Test
+ public void testGetInstanceName() {
+ config = createMock(ClientConfiguration.class);
+ mockIdConstruction(config);
+ replay(config);
+ zki = new ZooKeeperInstance(config, zcf);
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
+ List<String> children = new java.util.ArrayList<String>();
+ children.add("child1");
+ children.add("child2");
+ expect(zc.getChildren(Constants.ZROOT + Constants.ZINSTANCES)).andReturn(children);
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(Constants.UTF8));
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(UTF_8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(UTF_8));
+ replay(zc);
+ assertEquals("child2", zki.getInstanceName());
+ }
+
+ @Test
+ public void testAllZooKeepersAreUsed() {
+ final String zookeepers = "zk1,zk2,zk3", instanceName = "accumulo";
+ ZooCacheFactory factory = createMock(ZooCacheFactory.class);
+ expect(factory.getZooCache(zookeepers, 30000)).andReturn(zc).anyTimes();
+ replay(factory);
+ ClientConfiguration cfg = ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers);
+ ZooKeeperInstance zki = new ZooKeeperInstance(cfg, factory);
+ assertEquals(zookeepers, zki.getZooKeepers());
+ assertEquals(instanceName, zki.getInstanceName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index db4a12b,0000000..e2661e2
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -1,306 -1,0 +1,306 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client.mock;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.Test;
+
+public class MockNamespacesTest {
+
+ Random random = new Random();
+
+ /**
+ * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
+ */
+ @Test
+ public void testDefaultNamespace() throws Exception {
+ String tableName = "test";
+ Instance instance = new MockInstance("default");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
+ c.tableOperations().create(tableName);
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ /**
+ * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2"
+ * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the
+ * tables and delete the namespace.
+ */
+ @Test
+ public void testCreateAndDeleteNamespace() throws Exception {
+ String namespace = "testing";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ Instance instance = new MockInstance("createdelete");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().create(tableName1);
+ assertTrue(c.tableOperations().exists(tableName1));
+
+ c.tableOperations().create(tableName2);
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ // deleting
+ try {
+ // can't delete a namespace with tables in it
+ c.namespaceOperations().delete(namespace);
+ fail();
+ } catch (NamespaceNotEmptyException e) {
+ // ignore, supposed to happen
+ }
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ c.tableOperations().delete(tableName2);
+ assertTrue(!c.tableOperations().exists(tableName2));
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().delete(tableName1);
+ assertTrue(!c.tableOperations().exists(tableName1));
+ c.namespaceOperations().delete(namespace);
+ assertTrue(!c.namespaceOperations().exists(namespace));
+ }
+
+ /**
+ * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a
+ * namespace-wide level, use {@link NamespaceOperations}.
+ *
+ * Checks to make sure namespace-level properties are overridden by table-level properties.
+ *
+ * Checks to see if the default namespace's properties work as well.
+ */
+
+ @Test
+ public void testNamespaceProperties() throws Exception {
+ String namespace = "propchange";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ String propKey = Property.TABLE_SCAN_MAXMEM.getKey();
+ String propVal = "42K";
+
+ Instance instance = new MockInstance("props");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName1);
+ c.namespaceOperations().setProperty(namespace, propKey, propVal);
+
+ // check the namespace has the property
+ assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal));
+
+ // check that the table gets it from the namespace
+ assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
+
+ // test a second table to be sure the first wasn't magical
+ // (also, changed the order, the namespace has the property already)
+ c.tableOperations().create(tableName2);
+ assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
+
+ // test that table properties override namespace properties
+ String propKey2 = Property.TABLE_FILE_MAX.getKey();
+ String propVal2 = "42";
+ String tablePropVal = "13";
+
+ c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
+ c.namespaceOperations().setProperty("propchange", propKey2, propVal2);
+
+ assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
+
+ // now check that you can change the default namespace's properties
+ propVal = "13K";
+ String tableName = "some_table";
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal);
+
+ assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
+
+ // test the properties server-side by configuring an iterator.
+ // should not show anything with column-family = 'a'
+ String tableName3 = namespace + ".table3";
+ c.tableOperations().create(tableName3);
+
+ IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName());
+ c.namespaceOperations().attachIterator(namespace, setting);
+
+ BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes()));
+ bw.addMutation(m);
+ bw.flush();
+ bw.close();
+
+ // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY);
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+ }
+
+ /**
+ * This test renames and clones two separate table into different namespaces. different namespace.
+ */
+ @Test
+ public void testRenameAndCloneTableToNewNamespace() throws Exception {
+ String namespace1 = "renamed";
+ String namespace2 = "cloned";
+ String tableName = "table";
+ String tableName1 = "renamed.table1";
+ // String tableName2 = "cloned.table2";
+
+ Instance instance = new MockInstance("renameclone");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().create(namespace1);
+ c.namespaceOperations().create(namespace2);
+
+ c.tableOperations().rename(tableName, tableName1);
+
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(!c.tableOperations().exists(tableName));
+
+ // TODO implement clone in mock
+ /*
+ * c.tableOperations().clone(tableName1, tableName2, false, null, null);
+ *
+ * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2));
+ */
+ return;
+ }
+
+ /**
+ * This test renames a namespace and ensures that its tables are still correct
+ */
+ @Test
+ public void testNamespaceRename() throws Exception {
+ String namespace1 = "n1";
+ String namespace2 = "n2";
+ String table = "t";
+
+ Instance instance = new MockInstance("rename");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace1);
+ c.tableOperations().create(namespace1 + "." + table);
+
+ c.namespaceOperations().rename(namespace1, namespace2);
+
+ assertTrue(!c.namespaceOperations().exists(namespace1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(!c.tableOperations().exists(namespace1 + "." + table));
+ assertTrue(c.tableOperations().exists(namespace2 + "." + table));
+ }
+
+ /**
+ * This tests adding iterators to a namespace, listing them, and removing them
+ */
+ @Test
+ public void testNamespaceIterators() throws Exception {
+ Instance instance = new MockInstance("Iterators");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ String namespace = "iterator";
+ String tableName = namespace + ".table";
+ String iter = "thing";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName);
+
+ IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName());
+ HashSet<IteratorScope> scope = new HashSet<IteratorScope>();
+ scope.add(IteratorScope.scan);
+ c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope));
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
- m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
++ m.put("a", "b", new Value("abcde".getBytes(UTF_8)));
+ bw.addMutation(m);
+ bw.flush();
+
+ Scanner s = c.createScanner(tableName, Authorizations.EMPTY);
+ System.out.println(s.iterator().next());
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+
+ assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter));
+ c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope));
+ }
+
+ private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
+ for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException,
+ AccumuloSecurityException {
+ for (Entry<String,String> e : c.namespaceOperations().getProperties(n)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static class SimpleFilter extends Filter {
+ @Override
+ public boolean accept(Key k, Value v) {
+ if (k.getColumnFamily().toString().equals("a"))
+ return false;
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
index 17d2297,0000000..7bb3fa4
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
@@@ -1,129 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.security.tokens;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CredentialProviderTokenTest {
+
+ private static boolean isCredentialProviderAvailable = false;
+
+ // Keystore contains: {'root.password':'password', 'bob.password':'bob'}
+ private static String keystorePath;
+
+ @BeforeClass
+ public static void setup() {
+ try {
+ Class.forName(CredentialProviderFactoryShim.HADOOP_CRED_PROVIDER_CLASS_NAME);
+ isCredentialProviderAvailable = true;
+ } catch (Exception e) {
+ isCredentialProviderAvailable = false;
+ }
+
+ URL keystoreUrl = CredentialProviderTokenTest.class.getResource("/passwords.jceks");
+ Assert.assertNotNull(keystoreUrl);
+ keystorePath = "jceks://file/" + new File(keystoreUrl.getFile()).getAbsolutePath();
+ }
+
+ @Test
+ public void testPasswordsFromCredentialProvider() throws Exception {
+ if (!isCredentialProviderAvailable) {
+ return;
+ }
+
+ CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
- Assert.assertArrayEquals("password".getBytes(Constants.UTF8), token.getPassword());
++ Assert.assertArrayEquals("password".getBytes(UTF_8), token.getPassword());
+
+ token = new CredentialProviderToken("bob.password", keystorePath);
- Assert.assertArrayEquals("bob".getBytes(Constants.UTF8), token.getPassword());
++ Assert.assertArrayEquals("bob".getBytes(UTF_8), token.getPassword());
+ }
+
+ @Test
+ public void testEqualityAfterInit() throws Exception {
+ if (!isCredentialProviderAvailable) {
+ return;
+ }
+
+ CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
+
+ CredentialProviderToken uninitializedToken = new CredentialProviderToken();
+ Properties props = new Properties();
+ props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
+ props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
+ uninitializedToken.init(props);
+
+ Assert.assertArrayEquals(token.getPassword(), uninitializedToken.getPassword());
+ }
+
+ @Test
+ public void testMissingClassesThrowsException() throws Exception {
+ if (isCredentialProviderAvailable) {
+ return;
+ }
+
+ try {
+ new CredentialProviderToken("root.password", keystorePath);
+ Assert.fail("Should fail to create CredentialProviderToken when classes are not available");
+ } catch (IOException e) {
+ // pass
+ }
+ }
+
+ @Test
+ public void cloneReturnsCorrectObject() throws Exception {
+ if (!isCredentialProviderAvailable) {
+ return;
+ }
+
+ CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
+ CredentialProviderToken clone = token.clone();
+
+ Assert.assertEquals(token, clone);
+ Assert.assertArrayEquals(token.getPassword(), clone.getPassword());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingProperties() throws Exception {
+ CredentialProviderToken token = new CredentialProviderToken();
+ token.init(new Properties());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingNameProperty() throws Exception {
+ CredentialProviderToken token = new CredentialProviderToken();
+ Properties props = new Properties();
+ props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
+ token.init(props);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingProviderProperty() throws Exception {
+ CredentialProviderToken token = new CredentialProviderToken();
+ Properties props = new Properties();
+ props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
+ token.init(props);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
index 3570610,0000000..d9a47d7
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
@@@ -1,251 -1,0 +1,251 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConditionTest {
+ private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+ private static final String FAMILY = "family";
+ private static final String QUALIFIER = "qualifier";
+ private static final String VISIBILITY = "visibility";
+ private static final String VALUE = "value";
+ private static final IteratorSetting[] ITERATORS = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
+ new IteratorSetting(3, "third", "yetanotherclass")};
+
+ private String toString(ByteSequence bs) {
+ if (bs == null) {
+ return null;
+ }
- return new String(bs.toArray(), Constants.UTF8);
++ return new String(bs.toArray(), UTF_8);
+ }
+
+ private Condition c;
+
+ @Before
+ public void setUp() throws Exception {
+ c = new Condition(FAMILY, QUALIFIER);
+ }
+
+ @Test
+ public void testConstruction_CharSequence() {
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testConstruction_ByteArray() {
- c = new Condition(FAMILY.getBytes(Constants.UTF8), QUALIFIER.getBytes(Constants.UTF8));
++ c = new Condition(FAMILY.getBytes(UTF_8), QUALIFIER.getBytes(UTF_8));
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testConstruction_Text() {
+ c = new Condition(new Text(FAMILY), new Text(QUALIFIER));
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testConstruction_ByteSequence() {
- c = new Condition(new ArrayByteSequence(FAMILY.getBytes(Constants.UTF8)), new ArrayByteSequence(QUALIFIER.getBytes(Constants.UTF8)));
++ c = new Condition(new ArrayByteSequence(FAMILY.getBytes(UTF_8)), new ArrayByteSequence(QUALIFIER.getBytes(UTF_8)));
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testGetSetTimestamp() {
+ c.setTimestamp(1234L);
+ assertEquals(Long.valueOf(1234L), c.getTimestamp());
+ }
+
+ @Test
+ public void testSetValue_CharSequence() {
+ c.setValue(VALUE);
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testSetValue_ByteArray() {
- c.setValue(VALUE.getBytes(Constants.UTF8));
++ c.setValue(VALUE.getBytes(UTF_8));
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testSetValue_Text() {
+ c.setValue(new Text(VALUE));
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testSetValue_ByteSequence() {
- c.setValue(new ArrayByteSequence(VALUE.getBytes(Constants.UTF8)));
++ c.setValue(new ArrayByteSequence(VALUE.getBytes(UTF_8)));
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testGetSetVisibility() {
+ ColumnVisibility vis = new ColumnVisibility(VISIBILITY);
+ c.setVisibility(vis);
+ assertEquals(VISIBILITY, toString(c.getVisibility()));
+ }
+
+ @Test
+ public void testGetSetIterators() {
+ c.setIterators(ITERATORS);
+ assertArrayEquals(ITERATORS, c.getIterators());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetIterators_DuplicateName() {
+ IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
+ new IteratorSetting(3, "first", "yetanotherclass")};
+ c.setIterators(iterators);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetIterators_DuplicatePriority() {
+ IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
+ new IteratorSetting(1, "third", "yetanotherclass")};
+ c.setIterators(iterators);
+ }
+
+ @Test
+ public void testEquals() {
+ ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
+ c.setVisibility(cvis);
+ c.setValue(VALUE);
+ c.setTimestamp(1234L);
+ c.setIterators(ITERATORS);
+
+ // reflexivity
+ assertTrue(c.equals(c));
+
+ // non-nullity
+ assertFalse(c.equals(null));
+
+ // symmetry
+ Condition c2 = new Condition(FAMILY, QUALIFIER);
+ c2.setVisibility(cvis);
+ c2.setValue(VALUE);
+ c2.setTimestamp(1234L);
+ c2.setIterators(ITERATORS);
+ assertTrue(c.equals(c2));
+ assertTrue(c2.equals(c));
+
+ Condition c3 = new Condition("nope", QUALIFIER);
+ c3.setVisibility(cvis);
+ c3.setValue(VALUE);
+ c3.setTimestamp(1234L);
+ c3.setIterators(ITERATORS);
+ assertFalse(c.equals(c3));
+ assertFalse(c3.equals(c));
+ c3 = new Condition(FAMILY, "nope");
+ c3.setVisibility(cvis);
+ c3.setValue(VALUE);
+ c3.setTimestamp(1234L);
+ c3.setIterators(ITERATORS);
+ assertFalse(c.equals(c3));
+ assertFalse(c3.equals(c));
+
+ c2.setVisibility(new ColumnVisibility("sekrit"));
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setVisibility(cvis);
+ c2.setValue(EMPTY);
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setValue(VALUE);
+ c2.setTimestamp(2345L);
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setTimestamp(1234L);
+ c2.setIterators(new IteratorSetting[0]);
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setIterators(ITERATORS);
+ assertTrue(c.equals(c2));
+ assertTrue(c2.equals(c));
+
+ // set everything but vis, so its null
+ Condition c4 = new Condition(FAMILY, QUALIFIER);
+ c4.setValue(VALUE);
+ c4.setTimestamp(1234L);
+ c4.setIterators(ITERATORS);
+
+ assertFalse(c.equals(c4));
+ assertFalse(c4.equals(c));
+
+ // set everything but timestamp, so its null
+ Condition c5 = new Condition(FAMILY, QUALIFIER);
+ c5.setVisibility(cvis);
+ c5.setValue(VALUE);
+ c5.setIterators(ITERATORS);
+
+ assertFalse(c.equals(c5));
+ assertFalse(c5.equals(c));
+
+ // set everything but value
+ Condition c6 = new Condition(FAMILY, QUALIFIER);
+ c6.setVisibility(cvis);
+ c6.setTimestamp(1234L);
+ c6.setIterators(ITERATORS);
+
+ assertFalse(c.equals(c6));
+ assertFalse(c6.equals(c));
+
+ // test w/ no optional fields set
+ Condition c7 = new Condition(FAMILY, QUALIFIER);
+ Condition c8 = new Condition(FAMILY, QUALIFIER);
+ assertTrue(c7.equals(c8));
+ assertTrue(c8.equals(c7));
+
+ }
+
+ @Test
+ public void testHashCode() {
+ ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
+ c.setVisibility(cvis);
+ c.setValue(VALUE);
+ c.setTimestamp(1234L);
+ c.setIterators(ITERATORS);
+ int hc1 = c.hashCode();
+
+ Condition c2 = new Condition(FAMILY, QUALIFIER);
+ c2.setVisibility(cvis);
+ c2.setValue(VALUE);
+ c2.setTimestamp(1234L);
+ c2.setIterators(ITERATORS);
+ assertTrue(c.equals(c2));
+ assertEquals(hc1, c2.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 47f44f1,0000000..908c391
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@@ -1,149 -1,0 +1,149 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConditionalMutationTest {
- private static final byte[] ROW = "row".getBytes(Constants.UTF8);
++ private static final byte[] ROW = "row".getBytes(UTF_8);
+ private static final String FAMILY = "family";
+ private static final String QUALIFIER = "qualifier";
+ private static final String QUALIFIER2 = "qualifier2";
+ private static final String QUALIFIER3 = "qualifier3";
+
+ private Condition c1, c2;
+ private ConditionalMutation cm;
+
+ @Before
+ public void setUp() throws Exception {
+ c1 = new Condition(FAMILY, QUALIFIER);
+ c2 = new Condition(FAMILY, QUALIFIER2);
+ assertFalse(c1.equals(c2));
+ cm = new ConditionalMutation(ROW, c1, c2);
+ }
+
+ @Test
+ public void testConstruction_ByteArray() {
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_ByteArray_StartAndLength() {
+ cm = new ConditionalMutation(ROW, 1, 1, c1, c2);
- assertArrayEquals("o".getBytes(Constants.UTF8), cm.getRow());
++ assertArrayEquals("o".getBytes(UTF_8), cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_Text() {
+ cm = new ConditionalMutation(new Text(ROW), c1, c2);
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_CharSequence() {
- cm = new ConditionalMutation(new String(ROW, Constants.UTF8), c1, c2);
++ cm = new ConditionalMutation(new String(ROW, UTF_8), c1, c2);
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_ByteSequence() {
+ cm = new ConditionalMutation(new ArrayByteSequence(ROW), c1, c2);
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testCopyConstructor() {
+ ConditionalMutation cm2 = new ConditionalMutation(cm);
+ assertArrayEquals(cm.getRow(), cm2.getRow());
+ assertEquals(cm.getConditions(), cm2.getConditions());
+ }
+
+ @Test
+ public void testAddCondition() {
+ Condition c3 = new Condition(FAMILY, QUALIFIER3);
+ cm.addCondition(c3);
+ List<Condition> cs = cm.getConditions();
+ assertEquals(3, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ assertEquals(c3, cs.get(2));
+ }
+
+ @Test
+ public void testEquals() {
+ // reflexivity
+ assertTrue(cm.equals(cm));
+
+ // non-nullity
+ assertFalse(cm.equals((Object) null));
+
+ // symmetry
+ ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
+ assertTrue(cm.equals(cm2));
+ assertTrue(cm2.equals(cm));
+
- ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(Constants.UTF8), c1, c2);
++ ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(UTF_8), c1, c2);
+ assertFalse(cm.equals(cm3));
+ cm3 = new ConditionalMutation(ROW, c2, c1);
+ assertFalse(cm.getConditions().equals(cm3.getConditions()));
+ assertFalse(cm.equals(cm3));
+ }
+
+ @Test
+ public void testEquals_Mutation() {
+ Mutation m = new Mutation(ROW);
+ assertFalse(m.equals(cm));
+ assertFalse(cm.equals(m));
+ }
+
+ @Test
+ public void testHashcode() {
+ ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
+ assertTrue(cm.equals(cm2));
+ assertEquals(cm2.hashCode(), cm.hashCode());
+ }
+}
[43/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
index 5590f7b,0000000..abe5e8c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatus.java
@@@ -1,223 -1,0 +1,223 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
+
+import java.io.DataOutputStream;
+import java.io.File;
- import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.logger.LogEvents;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+/**
+ *
+ */
+public class UnusedWalDoesntCloseReplicationStatus extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ cfg.setNumTservers(1);
+ }
+
+ @Test
+ public void test() throws Exception {
+ File accumuloDir = this.getCluster().getConfig().getAccumuloDir();
+ final Connector conn = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+
+ conn.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+ conn.tableOperations().create(tableName);
+
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ final int numericTableId = Integer.valueOf(tableId);
+ final int fakeTableId = numericTableId + 1;
+
+ Assert.assertNotNull("Did not find table ID", tableId);
+
+ conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ // just sleep
+ conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
+ ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
+
+ FileSystem fs = FileSystem.getLocal(new Configuration());
+ File tserverWalDir = new File(accumuloDir, ServerConstants.WAL_DIR + Path.SEPARATOR + "faketserver+port");
+ File tserverWal = new File(tserverWalDir, UUID.randomUUID().toString());
+ fs.mkdirs(new Path(tserverWalDir.getAbsolutePath()));
+
+ // Make a fake WAL with no data in it for our real table
+ FSDataOutputStream out = fs.create(new Path(tserverWal.getAbsolutePath()));
+
- out.write(DfsLogger.LOG_FILE_HEADER_V3.getBytes(StandardCharsets.UTF_8));
++ out.write(DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8));
+
+ DataOutputStream dos = new DataOutputStream(out);
+ dos.writeUTF("NullCryptoModule");
+
+ // Fake a single update WAL that has a mutation for another table
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ key.event = OPEN;
+ key.tserverSession = tserverWal.getAbsolutePath();
+ key.filename = tserverWal.getAbsolutePath();
+ key.write(out);
+ value.write(out);
+
+ key.event = LogEvents.DEFINE_TABLET;
+ key.tablet = new KeyExtent(new Text(Integer.toString(fakeTableId)), null, null);
+ key.seq = 1l;
+ key.tid = 1;
+
+ key.write(dos);
+ value.write(dos);
+
+ key.tablet = null;
+ key.event = LogEvents.MUTATION;
+ key.filename = tserverWal.getAbsolutePath();
+ value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
+
+ key.write(dos);
+ value.write(dos);
+
+ key.event = LogEvents.COMPACTION_START;
+ key.filename = accumuloDir.getAbsolutePath() + "/tables/" + fakeTableId + "/t-000001/A000001.rf";
+ value.mutations = Collections.emptyList();
+
+ key.write(dos);
+ value.write(dos);
+
+ key.event = LogEvents.COMPACTION_FINISH;
+ value.mutations = Collections.emptyList();
+
+ key.write(dos);
+ value.write(dos);
+
+ dos.close();
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("m");
+ m.put("m", "m", "M");
+ bw.addMutation(m);
+ bw.close();
+
+ log.info("State of metadata table after inserting a record");
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ for (Entry<Key,Value> entry : s) {
+ System.out.println(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
+ }
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.ReplicationSection.getRange());
+ for (Entry<Key,Value> entry : s) {
+ System.out.println(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
+ }
+
+ log.info("Offline'ing table");
+
+ conn.tableOperations().offline(tableName, true);
+
+ // Add our fake WAL to the log column for this table
+ String walUri = tserverWal.toURI().toString();
+ KeyExtent extent = new KeyExtent(new Text(tableId), null, null);
+ bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ m = new Mutation(extent.getMetadataEntry());
+ m.put(MetadataSchema.TabletsSection.LogColumnFamily.NAME, new Text("localhost:12345/" + walUri),
- new Value((walUri + "|1").getBytes(StandardCharsets.UTF_8)));
++ new Value((walUri + "|1").getBytes(UTF_8)));
+ bw.addMutation(m);
+
+ // Add a replication entry for our fake WAL
+ m = new Mutation(MetadataSchema.ReplicationSection.getRowPrefix() + new Path(walUri).toString());
+ m.put(MetadataSchema.ReplicationSection.COLF, new Text(tableId), new Value(StatusUtil.fileCreated(System.currentTimeMillis()).toByteArray()));
+ bw.addMutation(m);
+ bw.close();
+
+ log.info("State of metadata after injecting WAL manually");
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ for (Entry<Key,Value> entry : s) {
+ log.info(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
+ }
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.ReplicationSection.getRange());
+ for (Entry<Key,Value> entry : s) {
+ log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
+ }
+
+ log.info("Bringing table online");
+ conn.tableOperations().online(tableName, true);
+
+ Assert.assertEquals(1, Iterables.size(conn.createScanner(tableName, Authorizations.EMPTY)));
+
+ log.info("Table has performed recovery, state of metadata:");
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ for (Entry<Key,Value> entry : s) {
+ log.info(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
+ }
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.ReplicationSection.getRange());
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(status));
+ Assert.assertFalse("Status record was closed and it should not be", status.getClosed());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
----------------------------------------------------------------------
diff --cc trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
index 0ed92fc,5010801..84e3204
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/receivers/ZooSpanClient.java
@@@ -16,8 -16,8 +16,9 @@@
*/
package org.apache.accumulo.trace.instrument.receivers;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
import java.io.IOException;
--import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@@ -39,7 -39,7 +40,6 @@@ public class ZooSpanClient extends Send
private static final Logger log = Logger.getLogger(ZooSpanClient.class);
private static final int TOTAL_TIME_WAIT_CONNECT_MS = 10 * 1000;
private static final int TIME_WAIT_CONNECT_CHECK_MS = 100;
- private static final Charset UTF8 = Charset.forName("UTF-8");
- private static final Charset UTF_8 = Charset.forName("UTF-8");
ZooKeeper zoo = null;
final String path;
[13/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 71b7155,0000000..78f8c7b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -1,318 -1,0 +1,320 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tables;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.security.SecurityPermission;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+
+public class TableManager {
+ private static SecurityPermission TABLE_MANAGER_PERMISSION = new SecurityPermission("tableManagerPermission");
+
+ private static final Logger log = Logger.getLogger(TableManager.class);
+ private static final Set<TableObserver> observers = Collections.synchronizedSet(new HashSet<TableObserver>());
+ private static final Map<String,TableState> tableStateCache = Collections.synchronizedMap(new HashMap<String,TableState>());
+ private static final byte[] ZERO_BYTE = new byte[] {'0'};
+
+ private static TableManager tableManager = null;
+
+ private final Instance instance;
+ private ZooCache zooStateCache;
+
+ public static void prepareNewNamespaceState(String instanceId, String namespaceId, String namespace, NodeExistsPolicy existsPolicy) throws KeeperException,
+ InterruptedException {
+ log.debug("Creating ZooKeeper entries for new namespace " + namespace + " (ID: " + namespaceId + ")");
+ String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zPath, new byte[0], existsPolicy);
- zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(UTF_8), existsPolicy);
+ zoo.putPersistentData(zPath + Constants.ZNAMESPACE_CONF, new byte[0], existsPolicy);
+ }
+
+ public static void prepareNewTableState(String instanceId, String tableId, String namespaceId, String tableName, TableState state,
+ NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ // state gets created last
+ log.debug("Creating ZooKeeper entries for new table " + tableName + " (ID: " + tableId + ") in namespace (ID: " + namespaceId + ")");
+ Pair<String,String> qualifiedTableName = Tables.qualify(tableName);
+ tableName = qualifiedTableName.getSecond();
+ String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zTablePath, new byte[0], existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_CONF, new byte[0], existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(Constants.UTF8), existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(UTF_8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(UTF_8), existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE, existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE, existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID, ZERO_BYTE, existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(UTF_8), existsPolicy);
+ }
+
+ public synchronized static TableManager getInstance() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TABLE_MANAGER_PERMISSION);
+ }
+ if (tableManager == null)
+ tableManager = new TableManager();
+ return tableManager;
+ }
+
+ private TableManager() {
+ instance = HdfsZooInstance.getInstance();
+ zooStateCache = new ZooCache(new TableStateWatcher());
+ updateTableStateCache();
+ }
+
+ public TableState getTableState(String tableId) {
+ return tableStateCache.get(tableId);
+ }
+
+ public static class IllegalTableTransitionException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ final TableState oldState;
+ final TableState newState;
+
+ public IllegalTableTransitionException(TableState oldState, TableState newState) {
+ this.oldState = oldState;
+ this.newState = newState;
+ }
+
+ public TableState getOldState() {
+ return oldState;
+ }
+
+ public TableState getNewState() {
+ return newState;
+ }
+
+ }
+
+ public synchronized void transitionTableState(final String tableId, final TableState newState) {
+ String statePath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE;
+
+ try {
- ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(Constants.UTF8), ZooUtil.PUBLIC, new Mutator() {
++ ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(UTF_8), ZooUtil.PUBLIC, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] oldData) throws Exception {
+ TableState oldState = TableState.UNKNOWN;
+ if (oldData != null)
- oldState = TableState.valueOf(new String(oldData, Constants.UTF8));
++ oldState = TableState.valueOf(new String(oldData, UTF_8));
+ boolean transition = true;
+ // +--------+
+ // v |
+ // NEW -> (ONLINE|OFFLINE)+--- DELETING
+ switch (oldState) {
+ case NEW:
+ transition = (newState == TableState.OFFLINE || newState == TableState.ONLINE);
+ break;
+ case ONLINE: // fall-through intended
+ case UNKNOWN:// fall through intended
+ case OFFLINE:
+ transition = (newState != TableState.NEW);
+ break;
+ case DELETING:
+ // Can't transition to any state from DELETING
+ transition = false;
+ break;
+ }
+ if (!transition)
+ throw new IllegalTableTransitionException(oldState, newState);
+ log.debug("Transitioning state for table " + tableId + " from " + oldState + " to " + newState);
- return newState.name().getBytes(Constants.UTF8);
++ return newState.name().getBytes(UTF_8);
+ }
+ });
+ } catch (Exception e) {
+ log.fatal("Failed to transition table to state " + newState);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void updateTableStateCache() {
+ synchronized (tableStateCache) {
+ for (String tableId : zooStateCache.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES))
+ if (zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE) != null)
+ updateTableStateCache(tableId);
+ }
+ }
+
+ public TableState updateTableStateCache(String tableId) {
+ synchronized (tableStateCache) {
+ TableState tState = TableState.UNKNOWN;
+ byte[] data = zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE);
+ if (data != null) {
- String sState = new String(data, Constants.UTF8);
++ String sState = new String(data, UTF_8);
+ try {
+ tState = TableState.valueOf(sState);
+ } catch (IllegalArgumentException e) {
+ log.error("Unrecognized state for table with tableId=" + tableId + ": " + sState);
+ }
+ tableStateCache.put(tableId, tState);
+ }
+ return tState;
+ }
+ }
+
+ public void addTable(String tableId, String namespaceId, String tableName, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException,
+ NamespaceNotFoundException {
+ prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
+ updateTableStateCache(tableId);
+ }
+
+ public void cloneTable(String srcTable, String tableId, String tableName, String namespaceId, Map<String,String> propertiesToSet,
+ Set<String> propertiesToExclude, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
+
+ String srcTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + srcTable + Constants.ZTABLE_CONF;
+ String newTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF;
+ ZooReaderWriter.getInstance().recursiveCopyPersistent(srcTablePath, newTablePath, NodeExistsPolicy.OVERWRITE);
+
+ for (Entry<String,String> entry : propertiesToSet.entrySet())
+ TablePropUtil.setTableProperty(tableId, entry.getKey(), entry.getValue());
+
+ for (String prop : propertiesToExclude)
+ ZooReaderWriter.getInstance().recursiveDelete(
+ Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF + "/" + prop, NodeMissingPolicy.SKIP);
+
+ updateTableStateCache(tableId);
+ }
+
+ public void removeTable(String tableId) throws KeeperException, InterruptedException {
+ synchronized (tableStateCache) {
+ tableStateCache.remove(tableId);
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE,
+ NodeMissingPolicy.SKIP);
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId, NodeMissingPolicy.SKIP);
+ }
+ }
+
+ public boolean addObserver(TableObserver to) {
+ synchronized (observers) {
+ synchronized (tableStateCache) {
+ to.initialize(Collections.unmodifiableMap(tableStateCache));
+ return observers.add(to);
+ }
+ }
+ }
+
+ public boolean removeObserver(TableObserver to) {
+ return observers.remove(to);
+ }
+
+ private class TableStateWatcher implements Watcher {
+ @Override
+ public void process(WatchedEvent event) {
+ if (log.isTraceEnabled())
+ log.trace(event);
+
+ final String zPath = event.getPath();
+ final EventType zType = event.getType();
+
+ String tablesPrefix = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ String tableId = null;
+
+ if (zPath != null && zPath.startsWith(tablesPrefix + "/")) {
+ String suffix = zPath.substring(tablesPrefix.length() + 1);
+ if (suffix.contains("/")) {
+ String[] sa = suffix.split("/", 2);
+ if (Constants.ZTABLE_STATE.equals("/" + sa[1]))
+ tableId = sa[0];
+ }
+ if (tableId == null) {
+ log.warn("Unknown path in " + event);
+ return;
+ }
+ }
+
+ switch (zType) {
+ case NodeChildrenChanged:
+ if (zPath != null && zPath.equals(tablesPrefix)) {
+ updateTableStateCache();
+ } else {
+ log.warn("Unexpected path " + zPath);
+ }
+ break;
+ case NodeCreated:
+ case NodeDataChanged:
+ // state transition
+ TableState tState = updateTableStateCache(tableId);
+ log.debug("State transition to " + tState + " @ " + event);
+ synchronized (observers) {
+ for (TableObserver to : observers)
+ to.stateChanged(tableId, tState);
+ }
+ break;
+ case NodeDeleted:
+ if (zPath != null
+ && tableId != null
+ && (zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_STATE) || zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_CONF) || zPath
+ .equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_NAME)))
+ tableStateCache.remove(tableId);
+ break;
+ case None:
+ switch (event.getState()) {
+ case Expired:
+ if (log.isTraceEnabled())
+ log.trace("Session expired " + event);
+ synchronized (observers) {
+ for (TableObserver to : observers)
+ to.sessionExpired();
+ }
+ break;
+ case SyncConnected:
+ default:
+ if (log.isTraceEnabled())
+ log.trace("Ignored " + event);
+ }
+ break;
+ default:
+ log.warn("Unandled " + event);
+ }
+ }
+ }
+
+ public void removeNamespace(String namespaceId) throws KeeperException, InterruptedException {
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId, NodeMissingPolicy.SKIP);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index eabdc0d,0000000..e10c0c1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -1,79 -1,0 +1,81 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tablets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Random;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+/**
+ * Allocates unique names for an accumulo instance. The names are unique for the lifetime of the instance.
+ *
+ * This is useful for filenames because it makes caching easy.
+ *
+ */
+
+public class UniqueNameAllocator {
+ private long next = 0;
+ private long maxAllocated = 0;
+ private String nextNamePath;
+ private Random rand;
+
+ private UniqueNameAllocator() {
+ nextNamePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZNEXT_FILE;
+ rand = new Random();
+ }
+
+ public synchronized String getNextName() {
+
+ while (next >= maxAllocated) {
+ final int allocate = 100 + rand.nextInt(100);
+
+ try {
+ byte[] max = ZooReaderWriter.getInstance().mutate(nextNamePath, null, ZooUtil.PRIVATE, new ZooReaderWriter.Mutator() {
+ public byte[] mutate(byte[] currentValue) throws Exception {
- long l = Long.parseLong(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++ long l = Long.parseLong(new String(currentValue, UTF_8), Character.MAX_RADIX);
+ l += allocate;
- return Long.toString(l, Character.MAX_RADIX).getBytes(Constants.UTF8);
++ return Long.toString(l, Character.MAX_RADIX).getBytes(UTF_8);
+ }
+ });
+
- maxAllocated = Long.parseLong(new String(max, Constants.UTF8), Character.MAX_RADIX);
++ maxAllocated = Long.parseLong(new String(max, UTF_8), Character.MAX_RADIX);
+ next = maxAllocated - allocate;
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
- return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), Constants.UTF8);
++ return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), UTF_8);
+ }
+
+ private static UniqueNameAllocator instance = null;
+
+ public static synchronized UniqueNameAllocator getInstance() {
+ if (instance == null)
+ instance = new UniqueNameAllocator();
+
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index cdff21d,0000000..198b2d6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,137 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class AddFilesWithMissingEntries {
+
+ static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names = "-update", description = "Make changes to the " + MetadataTable.NAME + " table to include missing files")
+ boolean update = false;
+ }
+
+ /**
+ * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who knows what
+ * they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
+
+ final Scanner scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+
+ KeyExtent last = new KeyExtent();
+ String directory = null;
+ Set<String> knownFiles = new HashSet<String>();
+
+ int count = 0;
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
+
+ // collect the list of known files and the directory for each extent
+ for (Entry<Key,Value> entry : scanner) {
+ Key key = entry.getKey();
+ KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
+ // when the key extent changes
+ if (!ke.equals(last)) {
+ if (directory != null) {
+ // add any files in the directory unknown to the key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ directory = null;
+ knownFiles.clear();
+ last = ke;
+ }
+ if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
+ directory = entry.getValue().toString();
+ log.debug("Found directory " + directory + " for row " + key.getRow().toString());
+ } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
+ String filename = key.getColumnQualifier().toString();
+ knownFiles.add(filename);
+ log.debug("METADATA file found: " + filename);
+ }
+ }
+ if (directory != null) {
+ // catch the last key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ log.info("There were " + count + " files that are unknown to the metadata table");
+ writer.close();
+ }
+
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update)
+ throws Exception {
+ int count = 0;
+ final String tableId = ke.getTableId().toString();
+ final Text row = ke.getMetadataEntry();
+ log.info(row.toString());
+ for (String dir : ServerConstants.getTablesDirs()) {
+ final Path path = new Path(dir + "/" + tableId + directory);
+ for (FileStatus file : fs.listStatus(path)) {
+ if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
+ continue;
+ final String filename = directory + "/" + file.getPath().getName();
+ if (!knownFiles.contains(filename)) {
+ count++;
+ final Mutation m = new Mutation(row);
+ String size = Long.toString(file.getLen());
+ String entries = "1"; // lie
+ String value = size + "," + entries;
- m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
++ m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(UTF_8)));
+ if (update) {
+ writer.getBatchWriter(MetadataTable.NAME).addMutation(m);
+ }
+ }
+ }
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 2926a3f,0000000..7a1e1ba
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -1,166 -1,0 +1,167 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class ChangeSecret {
+
+ static class Opts extends ClientOpts {
+ @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
+ String oldPass;
+ @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
+ String newPass;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ List<String> argsList = new ArrayList<String>(args.length + 2);
+ argsList.add("--old");
+ argsList.add("--new");
+ argsList.addAll(Arrays.asList(args));
+ opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
+ VolumeManager fs = VolumeManagerImpl.get();
+ Instance inst = opts.getInstance();
+ if (!verifyAccumuloIsDown(inst, opts.oldPass))
+ System.exit(-1);
+ String instanceId = rewriteZooKeeperInstance(inst, opts.oldPass, opts.newPass);
+ updateHdfs(fs, inst, instanceId);
+ if (opts.oldPass != null) {
+ deleteInstance(inst, opts.oldPass);
+ }
+ System.out.println("New instance id is " + instanceId);
+ System.out.println("Be sure to put your new secret in accumulo-site.xml");
+ }
+
+ interface Visitor {
+ void visit(ZooReader zoo, String path) throws Exception;
+ }
+
+ private static void recurse(ZooReader zoo, String root, Visitor v) {
+ try {
+ v.visit(zoo, root);
+ for (String child : zoo.getChildren(root)) {
+ recurse(zoo, root + "/" + child, v);
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private static boolean verifyAccumuloIsDown(Instance inst, String oldPassword) {
+ ZooReader zooReader = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPassword);
+ String root = ZooUtil.getRoot(inst);
+ final List<String> ephemerals = new ArrayList<String>();
+ recurse(zooReader, root, new Visitor() {
+ public void visit(ZooReader zoo, String path) throws Exception {
+ Stat stat = zoo.getStatus(path);
+ if (stat.getEphemeralOwner() != 0)
+ ephemerals.add(path);
+ }
+ });
+ if (ephemerals.size() == 0) {
+ return true;
+ }
+
+ System.err.println("The following ephemeral nodes exist, something is still running:");
+ for (String path : ephemerals) {
+ System.err.println(path);
+ }
+ return false;
+ }
+
+ private static String rewriteZooKeeperInstance(final Instance inst, String oldPass, String newPass) throws Exception {
+ final ZooReaderWriter orig = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPass);
+ final IZooReaderWriter new_ = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), newPass);
+ final String newInstanceId = UUID.randomUUID().toString();
+ String root = ZooUtil.getRoot(inst);
+ recurse(orig, root, new Visitor() {
+ public void visit(ZooReader zoo, String path) throws Exception {
+ String newPath = path.replace(inst.getInstanceID(), newInstanceId);
+ byte[] data = zoo.getData(path, null);
+ List<ACL> acls = orig.getZooKeeper().getACL(path, new Stat());
+ if (acls.containsAll(Ids.READ_ACL_UNSAFE)) {
+ new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ } else {
+ // upgrade
+ if (acls.containsAll(Ids.OPEN_ACL_UNSAFE)) {
+ // make user nodes private, they contain the user's password
+ String parts[] = path.split("/");
+ if (parts[parts.length - 2].equals("users")) {
+ new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ } else {
+ // everything else can have the readable acl
+ new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ }
+ } else {
+ new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ }
+ }
+ }
+ });
+ String path = "/accumulo/instances/" + inst.getInstanceName();
+ orig.recursiveDelete(path, NodeMissingPolicy.SKIP);
- new_.putPersistentData(path, newInstanceId.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ new_.putPersistentData(path, newInstanceId.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ return newInstanceId;
+ }
+
+ private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
+ // Need to recreate the instanceId on all of them to keep consistency
+ for (Volume v : fs.getVolumes()) {
+ final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ if (!v.getFileSystem().delete(instanceId, true)) {
+ throw new IOException("Could not recursively delete " + instanceId);
+ }
+
+ if (!v.getFileSystem().mkdirs(instanceId)) {
+ throw new IOException("Could not create directory " + instanceId);
+ }
+
+ v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
+ }
+ }
+
+ private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
+ IZooReaderWriter orig = new ZooReaderWriter(origInstance.getZooKeepers(), origInstance.getZooKeepersSessionTimeOut(), oldPass);
+ orig.recursiveDelete("/accumulo/" + origInstance.getInstanceID(), NodeMissingPolicy.SKIP);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index f2074a1,0000000..809a7c8
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -1,87 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class CleanZookeeper {
+
+ private static final Logger log = Logger.getLogger(CleanZookeeper.class);
+
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
+ String keepers = "localhost:2181";
+ @Parameter(names={"--password"}, description="the system secret", password=true)
+ String auth;
+ }
+
+ /**
+ * @param args
+ * must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
+ * @throws IOException
+ * error connecting to accumulo or zookeeper
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(CleanZookeeper.class.getName(), args);
+
+ String root = Constants.ZROOT;
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (opts.auth != null) {
- zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(Constants.UTF8));
++ zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(UTF_8));
+ }
+
+ try {
+ for (String child : zk.getChildren(root)) {
+ if (Constants.ZINSTANCES.equals("/" + child)) {
+ for (String instanceName : zk.getChildren(root + Constants.ZINSTANCES)) {
+ String instanceNamePath = root + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] id = zk.getData(instanceNamePath, null);
- if (id != null && !new String(id, Constants.UTF8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
++ if (id != null && !new String(id, UTF_8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
+ try {
+ zk.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
+ } catch (KeeperException.NoAuthException ex) {
+ log.warn("Unable to delete " + instanceNamePath);
+ }
+ }
+ }
+ } else if (!child.equals(HdfsZooInstance.getInstance().getInstanceID())) {
+ String path = root + "/" + child;
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ } catch (KeeperException.NoAuthException ex) {
+ log.warn("Unable to delete " + path);
+ }
+ }
+ }
+ } catch (Exception ex) {
+ System.out.println("Error Occurred: " + ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 448da86,0000000..d40830f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -1,81 -1,0 +1,83 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class DeleteZooInstance {
+
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
+ String instance;
+ }
+
+ static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
+ for (int i = 0; i < 10; i++){
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ return;
+ } catch (KeeperException.NotEmptyException ex) {
+ // ignored
+ } catch (Exception ex) {
+ throw ex;
+ }
+ }
+ }
+
+ /**
+ * @param args
+ * : the name or UUID of the instance to be deleted
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(DeleteZooInstance.class.getName(), args);
+
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // try instance name:
+ Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
+ Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
+ uuids.remove("instances");
+ if (instances.contains(opts.instance)) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
+ byte[] data = zk.getData(path, null);
+ deleteRetry(zk, path);
- deleteRetry(zk, Constants.ZROOT + "/" + new String(data, Constants.UTF8));
++ deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
+ } else if (uuids.contains(opts.instance)) {
+ // look for the real instance name
+ for (String instance : instances) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
+ byte[] data = zk.getData(path, null);
- if (opts.instance.equals(new String(data, Constants.UTF8)))
++ if (opts.instance.equals(new String(data, UTF_8)))
+ deleteRetry(zk, path);
+ }
+ deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 504956f,0000000..43348df
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,121 -1,0 +1,122 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class DumpZookeeper {
+
+ static IZooReaderWriter zk = null;
+
+ private static final Logger log = Logger.getLogger(DumpZookeeper.class);
+
+ private static class Encoded {
+ public String encoding;
+ public String value;
+
+ Encoded(String e, String v) {
+ encoding = e;
+ value = v;
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = "--root", description = "the root of the znode tree to dump")
+ String root = "/";
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
+ Logger.getRootLogger().setLevel(Level.WARN);
+ PrintStream out = System.out;
+ try {
+ zk = ZooReaderWriter.getInstance();
+
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
+ if (!child.equals("zookeeper"))
+ dump(out, opts.root, child, 1);
+ write(out, 0, "</dump>");
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ String path = root + "/" + child;
+ if (root.endsWith("/"))
+ path = root + child;
+ Stat stat = zk.getStatus(path);
+ if (stat == null)
+ return;
+ String type = "node";
+ if (stat.getEphemeralOwner() != 0) {
+ type = "ephemeral";
+ }
+ if (stat.getNumChildren() == 0) {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'/>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
+ }
+ } else {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
+ }
+ for (String c : zk.getChildren(path, null)) {
+ dump(out, path, c, indent + 1);
+ }
+ write(out, indent, "</node>");
+ }
+ }
+
+ private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ byte[] data = zk.getData(path, null);
+ for (int i = 0; i < data.length; i++) {
+ // does this look like simple ascii?
+ if (data[i] < ' ' || data[i] > '~')
+ return new Encoded("base64", Base64.encodeBase64String(data));
+ }
- return new Encoded(Constants.UTF8.name(), new String(data, Constants.UTF8));
++ return new Encoded(UTF_8.name(), new String(data, UTF_8));
+ }
+
+ private static void write(PrintStream out, int indent, String fmt, Object... args) {
+ for (int i = 0; i < indent; i++)
+ out.print(" ");
+ out.println(String.format(fmt, args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f4655dc,0000000..be85b1b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -1,225 -1,0 +1,227 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Formattable;
+import java.util.Formatter;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class ListInstances {
+
+ private static final Logger log = Logger.getLogger(ListInstances.class);
+
+ private static final int NAME_WIDTH = 20;
+ private static final int UUID_WIDTH = 37;
+ private static final int MASTER_WIDTH = 30;
+
+ private static final int ZOOKEEPER_TIMER_MILLIS = 30 * 1000;
+
+ static class Opts extends Help {
+ @Parameter(names="--print-errors", description="display errors while listing instances")
+ boolean printErrors = false;
+ @Parameter(names="--print-all", description="print information for all instances, not just those with names")
+ boolean printAll = false;
+ @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact")
+ String keepers = null;
+ }
+ static Opts opts = new Opts();
+ static int errors = 0;
+
+ public static void main(String[] args) {
+ opts.parseArgs(ListInstances.class.getName(), args);
+
+ if (opts.keepers == null) {
+ opts.keepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
+ }
+
+ String keepers = opts.keepers;
+ boolean printAll = opts.printAll;
+ boolean printErrors = opts.printErrors;
+
+ listInstances(keepers, printAll, printErrors);
+
+ }
+
+ static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) {
+ errors = 0;
+
+ System.out.println("INFO : Using ZooKeepers " + keepers);
+ ZooReader rdr = new ZooReader(keepers, ZOOKEEPER_TIMER_MILLIS);
+ ZooCache cache = new ZooCache(keepers, ZOOKEEPER_TIMER_MILLIS);
+
+ TreeMap<String,UUID> instanceNames = getInstanceNames(rdr, printErrors);
+
+ System.out.println();
+ printHeader();
+
+ for (Entry<String,UUID> entry : instanceNames.entrySet()) {
+ printInstanceInfo(cache, entry.getKey(), entry.getValue(), printErrors);
+ }
+
+ TreeSet<UUID> instancedIds = getInstanceIDs(rdr, printErrors);
+ instancedIds.removeAll(instanceNames.values());
+
+ if (printAll) {
+ for (UUID uuid : instancedIds) {
+ printInstanceInfo(cache, null, uuid, printErrors);
+ }
+ } else if (instancedIds.size() > 0) {
+ System.out.println();
+ System.out.println("INFO : " + instancedIds.size() + " unamed instances were not printed, run with --print-all to see all instances");
+ } else {
+ System.out.println();
+ }
+
+ if (!printErrors && errors > 0) {
+ System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info");
+ }
+ }
+
+ private static class CharFiller implements Formattable {
+
+ char c;
+
+ CharFiller(char c) {
+ this.c = c;
+ }
+
+ @Override
+ public void formatTo(Formatter formatter, int flags, int width, int precision) {
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < width; i++)
+ sb.append(c);
+ formatter.format(sb.toString());
+ }
+
+ }
+
+ private static void printHeader() {
+ System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MASTER_WIDTH + "s%n", "Instance Name", "Instance ID", "Master");
+ System.out.printf("%" + (NAME_WIDTH + 1) + "s+%" + (UUID_WIDTH + 1) + "s+%" + (MASTER_WIDTH + 1) + "s%n", new CharFiller('-'), new CharFiller('-'),
+ new CharFiller('-'));
+
+ }
+
+ private static void printInstanceInfo(ZooCache cache, String instanceName, UUID iid, boolean printErrors) {
+ String master = getMaster(cache, iid, printErrors);
+ if (instanceName == null) {
+ instanceName = "";
+ }
+
+ if (master == null) {
+ master = "";
+ }
+
+ System.out.printf("%" + NAME_WIDTH + "s |%" + UUID_WIDTH + "s |%" + MASTER_WIDTH + "s%n", "\"" + instanceName + "\"", iid, master);
+ }
+
+ private static String getMaster(ZooCache cache, UUID iid, boolean printErrors) {
+
+ if (iid == null) {
+ return null;
+ }
+
+ try {
+ String masterLocPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK;
+ byte[] master = ZooLock.getLockData(cache, masterLocPath, null);
+ if (master == null) {
+ return null;
+ }
- return new String(master, Constants.UTF8);
++ return new String(master, UTF_8);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ return null;
+ }
+ }
+
+ private static TreeMap<String,UUID> getInstanceNames(ZooReader zk, boolean printErrors) {
+
+ String instancesPath = Constants.ZROOT + Constants.ZINSTANCES;
+
+ TreeMap<String,UUID> tm = new TreeMap<String,UUID>();
+
+ List<String> names;
+
+ try {
+ names = zk.getChildren(instancesPath);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ return tm;
+ }
+
+ for (String name : names) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ try {
- UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), Constants.UTF8));
++ UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), UTF_8));
+ tm.put(name, iid);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ tm.put(name, null);
+ }
+ }
+
+ return tm;
+ }
+
+ private static TreeSet<UUID> getInstanceIDs(ZooReader zk, boolean printErrors) {
+ TreeSet<UUID> ts = new TreeSet<UUID>();
+
+ try {
+ List<String> children = zk.getChildren(Constants.ZROOT);
+
+ for (String iid : children) {
+ if (iid.equals("instances"))
+ continue;
+ try {
+ ts.add(UUID.fromString(iid));
+ } catch (Exception e) {
+ log.error("Exception: " + e);
+ }
+ }
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ }
+
+ return ts;
+ }
+
+ private static void handleException(Exception e, boolean printErrors) {
+ if (printErrors) {
+ e.printStackTrace();
+ }
+
+ errors++;
+ }
+}
[45/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
----------------------------------------------------------------------
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
index e37d78a,0000000..d389c0b
mode 100644,000000..100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
@@@ -1,100 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.replication.ReplicaSystemHelper;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.replication.DistributedWorkQueueWorkAssignerHelper;
+import org.apache.hadoop.fs.Path;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class ReplicationProcessorTest {
+
+ @Test
+ public void peerTypeExtractionFromConfiguration() {
+ Instance inst = EasyMock.createMock(Instance.class);
+ VolumeManager fs = EasyMock.createMock(VolumeManager.class);
+ Credentials creds = new Credentials("foo", new PasswordToken("bar"));
+
+ Map<String,String> data = new HashMap<>();
+
+ String peerName = "peer";
+ String configuration = "java.lang.String,foo";
+ data.put(Property.REPLICATION_PEERS + peerName, configuration);
+ ConfigurationCopy conf = new ConfigurationCopy(data);
+
+ ReplicationProcessor proc = new ReplicationProcessor(inst, conf, fs, creds);
+
+ Assert.assertEquals(configuration, proc.getPeerType(peerName));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void noPeerConfigurationThrowsAnException() {
+ Instance inst = EasyMock.createMock(Instance.class);
+ VolumeManager fs = EasyMock.createMock(VolumeManager.class);
+ Credentials creds = new Credentials("foo", new PasswordToken("bar"));
+
+ Map<String,String> data = new HashMap<>();
+ ConfigurationCopy conf = new ConfigurationCopy(data);
+
+ ReplicationProcessor proc = new ReplicationProcessor(inst, conf, fs, creds);
+
+ proc.getPeerType("foo");
+ }
+
+ @Test
+ public void filesWhichMakeNoProgressArentReplicatedAgain() throws Exception {
+ ReplicaSystem replica = EasyMock.createMock(ReplicaSystem.class);
+ ReplicaSystemHelper helper = EasyMock.createMock(ReplicaSystemHelper.class);
+ ReplicationProcessor proc = EasyMock.createMockBuilder(ReplicationProcessor.class).addMockedMethods("getReplicaSystem", "doesFileExist", "getStatus", "getHelper").createMock();
+
+ ReplicationTarget target = new ReplicationTarget("peer", "1", "1");
+ Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
+ Path path = new Path("/accumulo");
+
+ String queueKey = DistributedWorkQueueWorkAssignerHelper.getQueueKey(path.toString(), target);
+
+ EasyMock.expect(proc.getReplicaSystem(target)).andReturn(replica);
+ EasyMock.expect(proc.getStatus(path.toString(), target)).andReturn(status);
+ EasyMock.expect(proc.doesFileExist(path, target)).andReturn(true);
+ EasyMock.expect(proc.getHelper()).andReturn(helper);
+ EasyMock.expect(replica.replicate(path, status, target, helper)).andReturn(status);
+
+ EasyMock.replay(replica, proc);
+
- proc.process(queueKey, path.toString().getBytes(StandardCharsets.UTF_8));
++ proc.process(queueKey, path.toString().getBytes(UTF_8));
+
+ EasyMock.verify(replica, proc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/Shell.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/Shell.java
index 2ebdb6c,0000000..a0ff17a
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@@ -1,1229 -1,0 +1,1231 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++import static java.nio.charset.StandardCharsets.ISO_8859_1;
++
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.nio.charset.Charset;
- import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import jline.console.ConsoleReader;
+import jline.console.UserInterruptException;
+import jline.console.history.FileHistory;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.ClientConfigurationHelper;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.TConstraintViolationSummary;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.format.BinaryFormatter;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.accumulo.core.util.format.Formatter;
+import org.apache.accumulo.core.util.format.FormatterFactory;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.shell.commands.AboutCommand;
+import org.apache.accumulo.shell.commands.AddAuthsCommand;
+import org.apache.accumulo.shell.commands.AddSplitsCommand;
+import org.apache.accumulo.shell.commands.AuthenticateCommand;
+import org.apache.accumulo.shell.commands.ByeCommand;
+import org.apache.accumulo.shell.commands.ClasspathCommand;
+import org.apache.accumulo.shell.commands.ClearCommand;
+import org.apache.accumulo.shell.commands.CloneTableCommand;
+import org.apache.accumulo.shell.commands.ClsCommand;
+import org.apache.accumulo.shell.commands.CompactCommand;
+import org.apache.accumulo.shell.commands.ConfigCommand;
+import org.apache.accumulo.shell.commands.ConstraintCommand;
+import org.apache.accumulo.shell.commands.CreateNamespaceCommand;
+import org.apache.accumulo.shell.commands.CreateTableCommand;
+import org.apache.accumulo.shell.commands.CreateUserCommand;
+import org.apache.accumulo.shell.commands.DUCommand;
+import org.apache.accumulo.shell.commands.DebugCommand;
+import org.apache.accumulo.shell.commands.DeleteCommand;
+import org.apache.accumulo.shell.commands.DeleteIterCommand;
+import org.apache.accumulo.shell.commands.DeleteManyCommand;
+import org.apache.accumulo.shell.commands.DeleteNamespaceCommand;
+import org.apache.accumulo.shell.commands.DeleteRowsCommand;
+import org.apache.accumulo.shell.commands.DeleteScanIterCommand;
+import org.apache.accumulo.shell.commands.DeleteShellIterCommand;
+import org.apache.accumulo.shell.commands.DeleteTableCommand;
+import org.apache.accumulo.shell.commands.DeleteUserCommand;
+import org.apache.accumulo.shell.commands.DropTableCommand;
+import org.apache.accumulo.shell.commands.DropUserCommand;
+import org.apache.accumulo.shell.commands.EGrepCommand;
+import org.apache.accumulo.shell.commands.ExecfileCommand;
+import org.apache.accumulo.shell.commands.ExitCommand;
+import org.apache.accumulo.shell.commands.ExportTableCommand;
+import org.apache.accumulo.shell.commands.ExtensionCommand;
+import org.apache.accumulo.shell.commands.FateCommand;
+import org.apache.accumulo.shell.commands.FlushCommand;
+import org.apache.accumulo.shell.commands.FormatterCommand;
+import org.apache.accumulo.shell.commands.GetAuthsCommand;
+import org.apache.accumulo.shell.commands.GetGroupsCommand;
+import org.apache.accumulo.shell.commands.GetSplitsCommand;
+import org.apache.accumulo.shell.commands.GrantCommand;
+import org.apache.accumulo.shell.commands.GrepCommand;
+import org.apache.accumulo.shell.commands.HelpCommand;
+import org.apache.accumulo.shell.commands.HiddenCommand;
+import org.apache.accumulo.shell.commands.HistoryCommand;
+import org.apache.accumulo.shell.commands.ImportDirectoryCommand;
+import org.apache.accumulo.shell.commands.ImportTableCommand;
+import org.apache.accumulo.shell.commands.InfoCommand;
+import org.apache.accumulo.shell.commands.InsertCommand;
+import org.apache.accumulo.shell.commands.InterpreterCommand;
+import org.apache.accumulo.shell.commands.ListCompactionsCommand;
+import org.apache.accumulo.shell.commands.ListIterCommand;
+import org.apache.accumulo.shell.commands.ListScansCommand;
+import org.apache.accumulo.shell.commands.ListShellIterCommand;
+import org.apache.accumulo.shell.commands.MaxRowCommand;
+import org.apache.accumulo.shell.commands.MergeCommand;
+import org.apache.accumulo.shell.commands.NamespacePermissionsCommand;
+import org.apache.accumulo.shell.commands.NamespacesCommand;
+import org.apache.accumulo.shell.commands.NoTableCommand;
+import org.apache.accumulo.shell.commands.OfflineCommand;
+import org.apache.accumulo.shell.commands.OnlineCommand;
+import org.apache.accumulo.shell.commands.OptUtil;
+import org.apache.accumulo.shell.commands.PasswdCommand;
+import org.apache.accumulo.shell.commands.PingCommand;
+import org.apache.accumulo.shell.commands.QuestionCommand;
+import org.apache.accumulo.shell.commands.QuitCommand;
+import org.apache.accumulo.shell.commands.QuotedStringTokenizer;
+import org.apache.accumulo.shell.commands.RenameNamespaceCommand;
+import org.apache.accumulo.shell.commands.RenameTableCommand;
+import org.apache.accumulo.shell.commands.RevokeCommand;
+import org.apache.accumulo.shell.commands.ScanCommand;
+import org.apache.accumulo.shell.commands.ScriptCommand;
+import org.apache.accumulo.shell.commands.SetAuthsCommand;
+import org.apache.accumulo.shell.commands.SetGroupsCommand;
+import org.apache.accumulo.shell.commands.SetIterCommand;
+import org.apache.accumulo.shell.commands.SetScanIterCommand;
+import org.apache.accumulo.shell.commands.SetShellIterCommand;
+import org.apache.accumulo.shell.commands.SleepCommand;
+import org.apache.accumulo.shell.commands.SystemPermissionsCommand;
+import org.apache.accumulo.shell.commands.TableCommand;
+import org.apache.accumulo.shell.commands.TablePermissionsCommand;
+import org.apache.accumulo.shell.commands.TablesCommand;
+import org.apache.accumulo.shell.commands.TraceCommand;
+import org.apache.accumulo.shell.commands.UserCommand;
+import org.apache.accumulo.shell.commands.UserPermissionsCommand;
+import org.apache.accumulo.shell.commands.UsersCommand;
+import org.apache.accumulo.shell.commands.WhoAmICommand;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingOptionException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.ParameterException;
+
+/**
+ * A convenient console interface to perform basic accumulo functions Includes auto-complete, help, and quoted strings with escape sequences
+ */
+public class Shell extends ShellOptions {
+ public static final Logger log = Logger.getLogger(Shell.class);
+ private static final Logger audit = Logger.getLogger(Shell.class.getName() + ".audit");
+
- public static final Charset CHARSET = StandardCharsets.ISO_8859_1;
++ public static final Charset CHARSET = ISO_8859_1;
+ public static final int NO_FIXED_ARG_LENGTH_CHECK = -1;
+ public static final String COMMENT_PREFIX = "#";
+ public static final String HISTORY_DIR_NAME = ".accumulo";
+ public static final String HISTORY_FILE_NAME = "shell_history.txt";
+ private static final String SHELL_DESCRIPTION = "Shell - Apache Accumulo Interactive Shell";
+
+ protected int exitCode = 0;
+ private String tableName;
+ protected Instance instance;
+ private Connector connector;
+ protected ConsoleReader reader;
+ private String principal;
+ private AuthenticationToken token;
+ private final Class<? extends Formatter> defaultFormatterClass = DefaultFormatter.class;
+ private final Class<? extends Formatter> binaryFormatterClass = BinaryFormatter.class;
+ public Map<String,List<IteratorSetting>> scanIteratorOptions = new HashMap<String,List<IteratorSetting>>();
+ public Map<String,List<IteratorSetting>> iteratorProfiles = new HashMap<String,List<IteratorSetting>>();
+
+ private Token rootToken;
+ public final Map<String,Command> commandFactory = new TreeMap<String,Command>();
+ public final Map<String,Command[]> commandGrouping = new TreeMap<String,Command[]>();
+ protected boolean configError = false;
+
+ // exit if true
+ private boolean exit = false;
+
+ // file to execute commands from
+ protected File execFile = null;
+ // single command to execute from the command line
+ protected String execCommand = null;
+ protected boolean verbose = true;
+
+ private boolean tabCompletion;
+ private boolean disableAuthTimeout;
+ private long authTimeout;
+ private long lastUserActivity = System.nanoTime();
+ private boolean logErrorsToConsole = false;
+ private PrintWriter writer = null;
+ private boolean masking = false;
+
+ public Shell() throws IOException {
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
+ }
+
+ public Shell(ConsoleReader reader, PrintWriter writer) {
+ super();
+ this.reader = reader;
+ this.writer = writer;
+ }
+
+ // Not for client use
+ public boolean config(String... args) {
+ ShellOptionsJC options = new ShellOptionsJC();
+ JCommander jc = new JCommander();
+
+ jc.setProgramName("accumulo shell");
+ jc.addObject(options);
+ try {
+ jc.parse(args);
+ } catch (ParameterException e) {
+ configError = true;
+ }
+
+ if (options.isHelpEnabled()) {
+ configError = true;
+ }
+
+ if (!configError && options.getUnrecognizedOptions() != null) {
+ configError = true;
+ logError("Unrecognized Options: " + options.getUnrecognizedOptions().toString());
+ }
+
+ if (configError) {
+ jc.usage();
+ return true;
+ }
+
+ setDebugging(options.isDebugEnabled());
+ authTimeout = TimeUnit.MINUTES.toNanos(options.getAuthTimeout());
+ disableAuthTimeout = options.isAuthTimeoutDisabled();
+
+ // get the options that were parsed
+ String user = options.getUsername();
+ String password = options.getPassword();
+
+ tabCompletion = !options.isTabCompletionDisabled();
+
+ // Use a fake (Mock), ZK, or HdfsZK Accumulo instance
+ setInstance(options);
+
+ // AuthenticationToken options
+ token = options.getAuthenticationToken();
+ Map<String,String> loginOptions = options.getTokenProperties();
+
+ // process default parameters if unspecified
+ try {
+ boolean hasToken = (token != null);
+ boolean hasTokenOptions = !loginOptions.isEmpty();
+
+ if (hasToken && password != null) {
+ throw new ParameterException("Can not supply '--pass' option with '--tokenClass' option");
+ }
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ reader.getTerminal().setEchoEnabled(true);
+ }
+ });
+
+ // Need either both a token and options, or neither, but not just one.
+ if (hasToken != hasTokenOptions) {
+ throw new ParameterException("Must supply either both or neither of '--tokenClass' and '--tokenProperty'");
+ } else if (hasToken) { // implied hasTokenOptions
+ // Fully qualified name so we don't shadow java.util.Properties
+ org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties props;
+ // and line wrap it because the package name is so long
+ props = new org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties();
+
+ props.putAllStrings(loginOptions);
+ token.init(props);
+ } else {
+ // Read password if the user explicitly asked for it, or didn't specify anything at all
+ if ("stdin".equals(password) || password == null) {
+ password = reader.readLine("Password: ", '*');
+ }
+
+ if (password == null) {
+ // User cancel, e.g. Ctrl-D pressed
+ throw new ParameterException("No password or token option supplied");
+ } else {
+ this.token = new PasswordToken(password);
+ }
+ }
+
+ if (!options.isFake()) {
+ ZooReader zr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ DistributedTrace.enable(instance, zr, "shell", InetAddress.getLocalHost().getHostName());
+ }
+
+ this.setTableName("");
+ this.principal = user;
+ connector = instance.getConnector(this.principal, token);
+
+ } catch (Exception e) {
+ printException(e);
+ configError = true;
+ }
+
+ // decide whether to execute commands from a file and quit
+ if (options.getExecFile() != null) {
+ execFile = options.getExecFile();
+ verbose = false;
+ } else if (options.getExecFileVerbose() != null) {
+ execFile = options.getExecFileVerbose();
+ verbose = true;
+ }
+ execCommand = options.getExecCommand();
+ if (execCommand != null) {
+ verbose = false;
+ }
+
+ rootToken = new Token();
+
+ Command[] dataCommands = {new DeleteCommand(), new DeleteManyCommand(), new DeleteRowsCommand(), new EGrepCommand(), new FormatterCommand(),
+ new InterpreterCommand(), new GrepCommand(), new ImportDirectoryCommand(), new InsertCommand(), new MaxRowCommand(), new ScanCommand()};
+ Command[] debuggingCommands = {new ClasspathCommand(), new DebugCommand(), new ListScansCommand(), new ListCompactionsCommand(), new TraceCommand(),
+ new PingCommand()};
+ Command[] execCommands = {new ExecfileCommand(), new HistoryCommand(), new ExtensionCommand(), new ScriptCommand()};
+ Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new QuitCommand()};
+ Command[] helpCommands = {new AboutCommand(), new HelpCommand(), new InfoCommand(), new QuestionCommand()};
+ Command[] iteratorCommands = {new DeleteIterCommand(), new DeleteScanIterCommand(), new ListIterCommand(), new SetIterCommand(), new SetScanIterCommand(),
+ new SetShellIterCommand(), new ListShellIterCommand(), new DeleteShellIterCommand()};
+ Command[] otherCommands = {new HiddenCommand()};
+ Command[] permissionsCommands = {new GrantCommand(), new RevokeCommand(), new SystemPermissionsCommand(), new TablePermissionsCommand(),
+ new UserPermissionsCommand(), new NamespacePermissionsCommand()};
+ Command[] stateCommands = {new AuthenticateCommand(), new ClsCommand(), new ClearCommand(), new FateCommand(), new NoTableCommand(), new SleepCommand(),
+ new TableCommand(), new UserCommand(), new WhoAmICommand()};
+ Command[] tableCommands = {new CloneTableCommand(), new ConfigCommand(), new CreateTableCommand(), new DeleteTableCommand(), new DropTableCommand(),
+ new DUCommand(), new ExportTableCommand(), new ImportTableCommand(), new OfflineCommand(), new OnlineCommand(), new RenameTableCommand(),
+ new TablesCommand(), new NamespacesCommand(), new CreateNamespaceCommand(), new DeleteNamespaceCommand(), new RenameNamespaceCommand()};
+ Command[] tableControlCommands = {new AddSplitsCommand(), new CompactCommand(), new ConstraintCommand(), new FlushCommand(), new GetGroupsCommand(),
+ new GetSplitsCommand(), new MergeCommand(), new SetGroupsCommand()};
+ Command[] userCommands = {new AddAuthsCommand(), new CreateUserCommand(), new DeleteUserCommand(), new DropUserCommand(), new GetAuthsCommand(),
+ new PasswdCommand(), new SetAuthsCommand(), new UsersCommand()};
+ commandGrouping.put("-- Writing, Reading, and Removing Data --", dataCommands);
+ commandGrouping.put("-- Debugging Commands -------------------", debuggingCommands);
+ commandGrouping.put("-- Shell Execution Commands -------------", execCommands);
+ commandGrouping.put("-- Exiting Commands ---------------------", exitCommands);
+ commandGrouping.put("-- Help Commands ------------------------", helpCommands);
+ commandGrouping.put("-- Iterator Configuration ---------------", iteratorCommands);
+ commandGrouping.put("-- Permissions Administration Commands --", permissionsCommands);
+ commandGrouping.put("-- Shell State Commands -----------------", stateCommands);
+ commandGrouping.put("-- Table Administration Commands --------", tableCommands);
+ commandGrouping.put("-- Table Control Commands ---------------", tableControlCommands);
+ commandGrouping.put("-- User Administration Commands ---------", userCommands);
+
+ for (Command[] cmds : commandGrouping.values()) {
+ for (Command cmd : cmds)
+ commandFactory.put(cmd.getName(), cmd);
+ }
+ for (Command cmd : otherCommands) {
+ commandFactory.put(cmd.getName(), cmd);
+ }
+ return configError;
+ }
+
+ /**
+ * Sets the instance used by the shell based on the given options.
+ *
+ * @param options
+ * shell options
+ */
+ protected void setInstance(ShellOptionsJC options) {
+ // should only be one set of instance options set
+ instance = null;
+ if (options.isFake()) {
+ instance = new MockInstance("fake");
+ } else {
+ String instanceName, hosts;
+ if (options.isHdfsZooInstance()) {
+ instanceName = hosts = null;
+ } else if (options.getZooKeeperInstance().size() > 0) {
+ List<String> zkOpts = options.getZooKeeperInstance();
+ instanceName = zkOpts.get(0);
+ hosts = zkOpts.get(1);
+ } else {
+ instanceName = options.getZooKeeperInstanceName();
+ hosts = options.getZooKeeperHosts();
+ }
+ try {
+ instance = getZooInstance(instanceName, hosts, options.getClientConfiguration());
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Unable to load client config from " + options.getClientConfigFile(), e);
+ }
+ }
+ }
+
+ /*
+ * Takes instanceName and keepers as separate arguments, rather than just packaged into the clientConfig, so that we can fail over to accumulo-site.xml or
+ * HDFS config if they're unspecified.
+ */
+ private static Instance getZooInstance(String instanceName, String keepers, ClientConfiguration clientConfig) {
+ UUID instanceId = null;
+ if (instanceName == null) {
+ instanceName = clientConfig.get(ClientProperty.INSTANCE_NAME);
+ }
+ if (instanceName == null || keepers == null) {
+ AccumuloConfiguration conf = SiteConfiguration.getInstance(ClientConfigurationHelper.convertClientConfig(DefaultConfiguration.getInstance(), clientConfig));
+ if (instanceName == null) {
+ Path instanceDir = new Path(VolumeConfiguration.getVolumeUris(conf)[0], "instance_id");
+ instanceId = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(instanceDir, conf));
+ }
+ if (keepers == null) {
+ keepers = conf.get(Property.INSTANCE_ZK_HOST);
+ }
+ }
+ if (instanceId != null) {
+ return new ZooKeeperInstance(clientConfig.withInstance(instanceId).withZkHosts(keepers));
+ } else {
+ return new ZooKeeperInstance(clientConfig.withInstance(instanceName).withZkHosts(keepers));
+ }
+ }
+
+ public Connector getConnector() {
+ return connector;
+ }
+
+ public Instance getInstance() {
+ return instance;
+ }
+
+ public ClassLoader getClassLoader(final CommandLine cl, final Shell shellState) throws AccumuloException, TableNotFoundException, AccumuloSecurityException,
+ IOException, FileSystemException {
+
+ boolean tables = cl.hasOption(OptUtil.tableOpt().getOpt()) || !shellState.getTableName().isEmpty();
+ boolean namespaces = cl.hasOption(OptUtil.namespaceOpt().getOpt());
+
+ String classpath = null;
+ Iterable<Entry<String,String>> tableProps;
+
+ if (namespaces) {
+ try {
+ tableProps = shellState.getConnector().namespaceOperations().getProperties(OptUtil.getNamespaceOpt(cl, shellState));
+ } catch (NamespaceNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ }
+ } else if (tables) {
+ tableProps = shellState.getConnector().tableOperations().getProperties(OptUtil.getTableOpt(cl, shellState));
+ } else {
+ throw new IllegalArgumentException("No table or namespace specified");
+ }
+ for (Entry<String,String> entry : tableProps) {
+ if (entry.getKey().equals(Property.TABLE_CLASSPATH.getKey())) {
+ classpath = entry.getValue();
+ }
+ }
+
+ ClassLoader classloader;
+
+ if (classpath != null && !classpath.equals("")) {
+ shellState.getConnector().instanceOperations().getSystemConfiguration().get(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + classpath);
+
+ try {
+ AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Map.Entry<String,String>>() {
+ @Override
+ public Iterator<Entry<String,String>> iterator() {
+ try {
+ return shellState.getConnector().instanceOperations().getSystemConfiguration().entrySet().iterator();
+ } catch (AccumuloException e) {
+ throw new RuntimeException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }));
+ } catch (IllegalStateException ise) {}
+
+ classloader = AccumuloVFSClassLoader.getContextManager().getClassLoader(classpath);
+ } else {
+ classloader = AccumuloVFSClassLoader.getClassLoader();
+ }
+ return classloader;
+ }
+
+ public static void main(String args[]) throws IOException {
+ Shell shell = new Shell();
+ try {
+ shell.config(args);
+
+ System.exit(shell.start());
+ } finally {
+ shell.shutdown();
+ }
+ }
+
+ public int start() throws IOException {
+ if (configError)
+ return 1;
+
+ String input;
+ if (isVerbose())
+ printInfo();
+
+ String home = System.getProperty("HOME");
+ if (home == null)
+ home = System.getenv("HOME");
+ String configDir = home + "/" + HISTORY_DIR_NAME;
+ String historyPath = configDir + "/" + HISTORY_FILE_NAME;
+ File accumuloDir = new File(configDir);
+ if (!accumuloDir.exists() && !accumuloDir.mkdirs())
+ log.warn("Unable to make directory for history at " + accumuloDir);
+ try {
+ final FileHistory history = new FileHistory(new File(historyPath));
+ reader.setHistory(history);
+ // Add shutdown hook to flush file history, per jline javadocs
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ history.flush();
+ } catch (IOException e) {
+ log.warn("Could not flush history to file.");
+ }
+ }
+ });
+ } catch (IOException e) {
+ log.warn("Unable to load history file at " + historyPath);
+ }
+
+ // Turn Ctrl+C into Exception instead of JVM exit
+ reader.setHandleUserInterrupt(true);
+
+ ShellCompletor userCompletor = null;
+
+ if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(execFile, StandardCharsets.UTF_8.name());
++ java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
+ try {
+ while (scanner.hasNextLine() && !hasExited()) {
+ execCommand(scanner.nextLine(), true, isVerbose());
+ }
+ } finally {
+ scanner.close();
+ }
+ } else if (execCommand != null) {
+ for (String command : execCommand.split("\n")) {
+ execCommand(command, true, isVerbose());
+ }
+ return exitCode;
+ }
+
+ while (true) {
+ try {
+ if (hasExited())
+ return exitCode;
+
+ // If tab completion is true we need to reset
+ if (tabCompletion) {
+ if (userCompletor != null)
+ reader.removeCompleter(userCompletor);
+
+ userCompletor = setupCompletion();
+ reader.addCompleter(userCompletor);
+ }
+
+ reader.setPrompt(getDefaultPrompt());
+ input = reader.readLine();
+ if (input == null) {
+ reader.println();
+ return exitCode;
+ } // User Canceled (Ctrl+D)
+
+ execCommand(input, disableAuthTimeout, false);
+ } catch (UserInterruptException uie) {
+ // User Cancelled (Ctrl+C)
+ reader.println();
+
+ String partialLine = uie.getPartialLine();
+ if (partialLine == null || "".equals(uie.getPartialLine().trim())) {
+ // No content, actually exit
+ return exitCode;
+ }
+ } finally {
+ reader.flush();
+ }
+ }
+ }
+
+ public void shutdown() {
+ if (reader != null) {
+ reader.shutdown();
+ }
+ }
+
+ public void printInfo() throws IOException {
+ reader.print("\n" + SHELL_DESCRIPTION + "\n" + "- \n" + "- version: " + Constants.VERSION + "\n" + "- instance name: "
+ + connector.getInstance().getInstanceName() + "\n" + "- instance id: " + connector.getInstance().getInstanceID() + "\n" + "- \n"
+ + "- type 'help' for a list of available commands\n" + "- \n");
+ reader.flush();
+ }
+
+ public void printVerboseInfo() throws IOException {
+ StringBuilder sb = new StringBuilder("-\n");
+ sb.append("- Current user: ").append(connector.whoami()).append("\n");
+ if (execFile != null)
+ sb.append("- Executing commands from: ").append(execFile).append("\n");
+ if (disableAuthTimeout)
+ sb.append("- Authorization timeout: disabled\n");
+ else
+ sb.append("- Authorization timeout: ").append(String.format("%ds%n", TimeUnit.NANOSECONDS.toSeconds(authTimeout)));
+ sb.append("- Debug: ").append(isDebuggingEnabled() ? "on" : "off").append("\n");
+ if (!scanIteratorOptions.isEmpty()) {
+ for (Entry<String,List<IteratorSetting>> entry : scanIteratorOptions.entrySet()) {
+ sb.append("- Session scan iterators for table ").append(entry.getKey()).append(":\n");
+ for (IteratorSetting setting : entry.getValue()) {
+ sb.append("- Iterator ").append(setting.getName()).append(" options:\n");
+ sb.append("- ").append("iteratorPriority").append(" = ").append(setting.getPriority()).append("\n");
+ sb.append("- ").append("iteratorClassName").append(" = ").append(setting.getIteratorClass()).append("\n");
+ for (Entry<String,String> optEntry : setting.getOptions().entrySet()) {
+ sb.append("- ").append(optEntry.getKey()).append(" = ").append(optEntry.getValue()).append("\n");
+ }
+ }
+ }
+ }
+ sb.append("-\n");
+ reader.print(sb.toString());
+ }
+
+ public String getDefaultPrompt() {
+ return connector.whoami() + "@" + connector.getInstance().getInstanceName() + (getTableName().isEmpty() ? "" : " ") + getTableName() + "> ";
+ }
+
+ public void execCommand(String input, boolean ignoreAuthTimeout, boolean echoPrompt) throws IOException {
+ audit.log(Level.INFO, getDefaultPrompt() + input);
+ if (echoPrompt) {
+ reader.print(getDefaultPrompt());
+ reader.println(input);
+ }
+
+ if (input.startsWith(COMMENT_PREFIX)) {
+ return;
+ }
+
+ String fields[];
+ try {
+ fields = new QuotedStringTokenizer(input).getTokens();
+ } catch (BadArgumentException e) {
+ printException(e);
+ ++exitCode;
+ return;
+ }
+ if (fields.length == 0)
+ return;
+
+ String command = fields[0];
+ fields = fields.length > 1 ? Arrays.copyOfRange(fields, 1, fields.length) : new String[] {};
+
+ Command sc = null;
+ if (command.length() > 0) {
+ try {
+ // Obtain the command from the command table
+ sc = commandFactory.get(command);
+ if (sc == null) {
+ reader.println(String.format("Unknown command \"%s\". Enter \"help\" for a list possible commands.", command));
+ reader.flush();
+ return;
+ }
+
+ long duration = System.nanoTime() - lastUserActivity;
+ if (!(sc instanceof ExitCommand) && !ignoreAuthTimeout && (duration < 0 || duration > authTimeout)) {
+ reader.println("Shell has been idle for too long. Please re-authenticate.");
+ boolean authFailed = true;
+ do {
+ String pwd = readMaskedLine("Enter current password for '" + connector.whoami() + "': ", '*');
+ if (pwd == null) {
+ reader.println();
+ return;
+ } // user canceled
+
+ try {
+ authFailed = !connector.securityOperations().authenticateUser(connector.whoami(), new PasswordToken(pwd));
+ } catch (Exception e) {
+ ++exitCode;
+ printException(e);
+ }
+
+ if (authFailed)
+ reader.print("Invalid password. ");
+ } while (authFailed);
+ lastUserActivity = System.nanoTime();
+ }
+
+ // Get the options from the command on how to parse the string
+ Options parseOpts = sc.getOptionsWithHelp();
+
+ // Parse the string using the given options
+ CommandLine cl = new BasicParser().parse(parseOpts, fields);
+
+ int actualArgLen = cl.getArgs().length;
+ int expectedArgLen = sc.numArgs();
+ if (cl.hasOption(helpOption)) {
+ // Display help if asked to; otherwise execute the command
+ sc.printHelp(this);
+ } else if (expectedArgLen != NO_FIXED_ARG_LENGTH_CHECK && actualArgLen != expectedArgLen) {
+ ++exitCode;
+ // Check for valid number of fixed arguments (if not
+ // negative; negative means it is not checked, for
+ // vararg-like commands)
+ printException(new IllegalArgumentException(String.format("Expected %d argument%s. There %s %d.", expectedArgLen, expectedArgLen == 1 ? "" : "s",
+ actualArgLen == 1 ? "was" : "were", actualArgLen)));
+ sc.printHelp(this);
+ } else {
+ int tmpCode = sc.execute(input, cl, this);
+ exitCode += tmpCode;
+ reader.flush();
+ }
+
+ } catch (ConstraintViolationException e) {
+ ++exitCode;
+ printConstraintViolationException(e);
+ } catch (TableNotFoundException e) {
+ ++exitCode;
+ if (getTableName().equals(e.getTableName()))
+ setTableName("");
+ printException(e);
+ } catch (ParseException e) {
+ // not really an error if the exception is a missing required
+ // option when the user is asking for help
+ if (!(e instanceof MissingOptionException && (Arrays.asList(fields).contains("-" + helpOption) || Arrays.asList(fields).contains("--" + helpLongOption)))) {
+ ++exitCode;
+ printException(e);
+ }
+ if (sc != null)
+ sc.printHelp(this);
+ } catch (UserInterruptException e) {
+ ++exitCode;
+ } catch (Exception e) {
+ ++exitCode;
+ printException(e);
+ }
+ } else {
+ ++exitCode;
+ printException(new BadArgumentException("Unrecognized empty command", command, -1));
+ }
+ reader.flush();
+ }
+
+ /**
+ * The command tree is built in reverse so that the references are more easily linked up. There is some code in token to allow forward building of the command
+ * tree.
+ */
+ private ShellCompletor setupCompletion() {
+ rootToken = new Token();
+
+ Set<String> tableNames = null;
+ try {
+ tableNames = connector.tableOperations().list();
+ } catch (Exception e) {
+ log.debug("Unable to obtain list of tables", e);
+ tableNames = Collections.emptySet();
+ }
+
+ Set<String> userlist = null;
+ try {
+ userlist = connector.securityOperations().listLocalUsers();
+ } catch (Exception e) {
+ log.debug("Unable to obtain list of users", e);
+ userlist = Collections.emptySet();
+ }
+
+ Set<String> namespaces = null;
+ try {
+ namespaces = connector.namespaceOperations().list();
+ } catch (Exception e) {
+ log.debug("Unable to obtain list of namespaces", e);
+ namespaces = Collections.emptySet();
+ }
+
+ Map<Command.CompletionSet,Set<String>> options = new HashMap<Command.CompletionSet,Set<String>>();
+
+ Set<String> commands = new HashSet<String>();
+ for (String a : commandFactory.keySet())
+ commands.add(a);
+
+ Set<String> modifiedUserlist = new HashSet<String>();
+ Set<String> modifiedTablenames = new HashSet<String>();
+ Set<String> modifiedNamespaces = new HashSet<String>();
+
+ for (String a : tableNames)
+ modifiedTablenames.add(a.replaceAll("([\\s'\"])", "\\\\$1"));
+ for (String a : userlist)
+ modifiedUserlist.add(a.replaceAll("([\\s'\"])", "\\\\$1"));
+ for (String a : namespaces) {
+ String b = a.replaceAll("([\\s'\"])", "\\\\$1");
+ modifiedNamespaces.add(b.isEmpty() ? "\"\"" : b);
+ }
+
+ options.put(Command.CompletionSet.USERNAMES, modifiedUserlist);
+ options.put(Command.CompletionSet.TABLENAMES, modifiedTablenames);
+ options.put(Command.CompletionSet.NAMESPACES, modifiedNamespaces);
+ options.put(Command.CompletionSet.COMMANDS, commands);
+
+ for (Command[] cmdGroup : commandGrouping.values()) {
+ for (Command c : cmdGroup) {
+ c.getOptionsWithHelp(); // prep the options for the command
+ // so that the completion can
+ // include them
+ c.registerCompletion(rootToken, options);
+ }
+ }
+ return new ShellCompletor(rootToken, options);
+ }
+
+ /**
+ * The Command class represents a command to be run in the shell. It contains the methods to execute along with some methods to help tab completion, and
+ * return the command name, help, and usage.
+ */
+ public static abstract class Command {
+ // Helper methods for completion
+ public enum CompletionSet {
+ TABLENAMES, USERNAMES, COMMANDS, NAMESPACES
+ }
+
+ static Set<String> getCommandNames(Map<CompletionSet,Set<String>> objects) {
+ return objects.get(CompletionSet.COMMANDS);
+ }
+
+ static Set<String> getTableNames(Map<CompletionSet,Set<String>> objects) {
+ return objects.get(CompletionSet.TABLENAMES);
+ }
+
+ static Set<String> getUserNames(Map<CompletionSet,Set<String>> objects) {
+ return objects.get(CompletionSet.USERNAMES);
+ }
+
+ static Set<String> getNamespaces(Map<CompletionSet,Set<String>> objects) {
+ return objects.get(CompletionSet.NAMESPACES);
+ }
+
+ public void registerCompletionGeneral(Token root, Set<String> args, boolean caseSens) {
+ Token t = new Token(args);
+ t.setCaseSensitive(caseSens);
+
+ Token command = new Token(getName());
+ command.addSubcommand(t);
+
+ root.addSubcommand(command);
+ }
+
+ public void registerCompletionForTables(Token root, Map<CompletionSet,Set<String>> completionSet) {
+ registerCompletionGeneral(root, completionSet.get(CompletionSet.TABLENAMES), true);
+ }
+
+ public void registerCompletionForUsers(Token root, Map<CompletionSet,Set<String>> completionSet) {
+ registerCompletionGeneral(root, completionSet.get(CompletionSet.USERNAMES), true);
+ }
+
+ public void registerCompletionForCommands(Token root, Map<CompletionSet,Set<String>> completionSet) {
+ registerCompletionGeneral(root, completionSet.get(CompletionSet.COMMANDS), false);
+ }
+
+ public void registerCompletionForNamespaces(Token root, Map<CompletionSet,Set<String>> completionSet) {
+ registerCompletionGeneral(root, completionSet.get(CompletionSet.NAMESPACES), true);
+ }
+
+ // abstract methods to override
+ public abstract int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception;
+
+ public abstract String description();
+
+ /**
+ * If the number of arguments is not always zero (not including those arguments handled through Options), make sure to override the {@link #usage()} method.
+ * Otherwise, {@link #usage()} does need to be overridden.
+ */
+ public abstract int numArgs();
+
+ // OPTIONAL methods to override:
+
+ // the general version of getname uses reflection to get the class name
+ // and then cuts off the suffix -Command to get the name of the command
+ public String getName() {
+ String s = this.getClass().getName();
+ int st = Math.max(s.lastIndexOf('$'), s.lastIndexOf('.'));
+ int i = s.indexOf("Command");
+ return i > 0 ? s.substring(st + 1, i).toLowerCase(Locale.ENGLISH) : null;
+ }
+
+ // The general version of this method adds the name
+ // of the command to the completion tree
+ public void registerCompletion(Token root, Map<CompletionSet,Set<String>> completion_set) {
+ root.addSubcommand(new Token(getName()));
+ }
+
+ // The general version of this method uses the HelpFormatter
+ // that comes with the apache Options package to print out the help
+ public final void printHelp(Shell shellState) {
+ shellState.printHelp(usage(), "description: " + this.description(), getOptionsWithHelp());
+ }
+
+ public final void printHelp(Shell shellState, int width) {
+ shellState.printHelp(usage(), "description: " + this.description(), getOptionsWithHelp(), width);
+ }
+
+ // Get options with help
+ public final Options getOptionsWithHelp() {
+ Options opts = getOptions();
+ opts.addOption(new Option(helpOption, helpLongOption, false, "display this help"));
+ return opts;
+ }
+
+ // General usage is just the command
+ public String usage() {
+ return getName();
+ }
+
+ // General Options are empty
+ public Options getOptions() {
+ return new Options();
+ }
+ }
+
+ public interface PrintLine {
+ void print(String s);
+
+ void close();
+ }
+
+ public static class PrintShell implements PrintLine {
+ ConsoleReader reader;
+
+ public PrintShell(ConsoleReader reader) {
+ this.reader = reader;
+ }
+
+ @Override
+ public void print(String s) {
+ try {
+ reader.println(s);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void close() {}
+ };
+
+ public static class PrintFile implements PrintLine {
+ PrintWriter writer;
+
+ public PrintFile(String filename) throws FileNotFoundException {
- writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), StandardCharsets.UTF_8)));
++ writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), UTF_8)));
+ }
+
+ @Override
+ public void print(String s) {
+ writer.println(s);
+ }
+
+ @Override
+ public void close() {
+ writer.close();
+ }
+ };
+
+ public final void printLines(Iterator<String> lines, boolean paginate) throws IOException {
+ printLines(lines, paginate, null);
+ }
+
+ public final void printLines(Iterator<String> lines, boolean paginate, PrintLine out) throws IOException {
+ int linesPrinted = 0;
+ String prompt = "-- hit any key to continue or 'q' to quit --";
+ int lastPromptLength = prompt.length();
+ int termWidth = reader.getTerminal().getWidth();
+ int maxLines = reader.getTerminal().getHeight();
+
+ String peek = null;
+ while (lines.hasNext()) {
+ String nextLine = lines.next();
+ if (nextLine == null)
+ continue;
+ for (String line : nextLine.split("\\n")) {
+ if (out == null) {
+ if (peek != null) {
+ reader.println(peek);
+ if (paginate) {
+ linesPrinted += peek.length() == 0 ? 0 : Math.ceil(peek.length() * 1.0 / termWidth);
+
+ // check if displaying the next line would result in
+ // scrolling off the screen
+ if (linesPrinted + Math.ceil(lastPromptLength * 1.0 / termWidth) + Math.ceil(prompt.length() * 1.0 / termWidth)
+ + Math.ceil(line.length() * 1.0 / termWidth) > maxLines) {
+ linesPrinted = 0;
+ int numdashes = (termWidth - prompt.length()) / 2;
+ String nextPrompt = repeat("-", numdashes) + prompt + repeat("-", numdashes);
+ lastPromptLength = nextPrompt.length();
+ reader.print(nextPrompt);
+ reader.flush();
+
+ if (Character.toUpperCase((char) reader.readCharacter()) == 'Q') {
+ reader.println();
+ return;
+ }
+ reader.println();
+ termWidth = reader.getTerminal().getWidth();
+ maxLines = reader.getTerminal().getHeight();
+ }
+ }
+ }
+ peek = line;
+ } else {
+ out.print(line);
+ }
+ }
+ }
+ if (out == null && peek != null) {
+ reader.println(peek);
+ }
+ }
+
+ public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, Class<? extends Formatter> formatterClass,
+ PrintLine outFile) throws IOException {
+ printLines(FormatterFactory.getFormatter(formatterClass, scanner, printTimestamps), paginate, outFile);
+ }
+
+ public final void printRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, Class<? extends Formatter> formatterClass)
+ throws IOException {
+ printLines(FormatterFactory.getFormatter(formatterClass, scanner, printTimestamps), paginate);
+ }
+
+ public final void printBinaryRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate, PrintLine outFile) throws IOException {
+ printLines(FormatterFactory.getFormatter(binaryFormatterClass, scanner, printTimestamps), paginate, outFile);
+ }
+
+ public final void printBinaryRecords(Iterable<Entry<Key,Value>> scanner, boolean printTimestamps, boolean paginate) throws IOException {
+ printLines(FormatterFactory.getFormatter(binaryFormatterClass, scanner, printTimestamps), paginate);
+ }
+
+ public static String repeat(String s, int c) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < c; i++)
+ sb.append(s);
+ return sb.toString();
+ }
+
+ public void checkTableState() {
+ if (getTableName().isEmpty())
+ throw new IllegalStateException(
+ "Not in a table context. Please use 'table <tableName>' to switch to a table, or use '-t' to specify a table if option is available.");
+ }
+
+ private final void printConstraintViolationException(ConstraintViolationException cve) {
+ printException(cve, "");
+ int COL1 = 50, COL2 = 14;
+ int col3 = Math.max(1, Math.min(Integer.MAX_VALUE, reader.getTerminal().getWidth() - COL1 - COL2 - 6));
+ logError(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s%n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
+ logError(String.format("%-" + COL1 + "s | %" + COL2 + "s | %-" + col3 + "s%n", "Constraint class", "Violation code", "Violation Description"));
+ logError(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s%n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
+ for (TConstraintViolationSummary cvs : cve.violationSummaries)
+ logError(String.format("%-" + COL1 + "s | %" + COL2 + "d | %-" + col3 + "s%n", cvs.constrainClass, cvs.violationCode, cvs.violationDescription));
+ logError(String.format("%" + COL1 + "s-+-%" + COL2 + "s-+-%" + col3 + "s%n", repeat("-", COL1), repeat("-", COL2), repeat("-", col3)));
+ }
+
+ public final void printException(Exception e) {
+ printException(e, e.getMessage());
+ }
+
+ private final void printException(Exception e, String msg) {
+ logError(e.getClass().getName() + (msg != null ? ": " + msg : ""));
+ log.debug(e.getClass().getName() + (msg != null ? ": " + msg : ""), e);
+ }
+
+ public static final void setDebugging(boolean debuggingEnabled) {
+ Logger.getLogger(Constants.CORE_PACKAGE_NAME).setLevel(debuggingEnabled ? Level.TRACE : Level.INFO);
+ }
+
+ public static final boolean isDebuggingEnabled() {
+ return Logger.getLogger(Constants.CORE_PACKAGE_NAME).isTraceEnabled();
+ }
+
+ private final void printHelp(String usage, String description, Options opts) {
+ printHelp(usage, description, opts, Integer.MAX_VALUE);
+ }
+
+ private final void printHelp(String usage, String description, Options opts, int width) {
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
+ }
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ public void resetExitCode() {
+ exitCode = 0;
+ }
+
+ public void setExit(boolean exit) {
+ this.exit = exit;
+ }
+
+ public boolean getExit() {
+ return this.exit;
+ }
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = (tableName == null || tableName.isEmpty()) ? "" : Tables.qualified(tableName);
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public ConsoleReader getReader() {
+ return reader;
+ }
+
+ public void updateUser(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ connector = instance.getConnector(principal, token);
+ this.principal = principal;
+ this.token = token;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ /**
+ * Return the formatter for the current table.
+ *
+ * @return the formatter class for the current table
+ */
+ public Class<? extends Formatter> getFormatter() {
+ return getFormatter(this.tableName);
+ }
+
+ /**
+ * Return the formatter for the given table.
+ *
+ * @param tableName
+ * the table name
+ * @return the formatter class for the given table
+ */
+ public Class<? extends Formatter> getFormatter(String tableName) {
+ Class<? extends Formatter> formatter = FormatterCommand.getCurrentFormatter(tableName, this);
+
+ if (null == formatter) {
+ logError("Could not load the specified formatter. Using the DefaultFormatter");
+ return this.defaultFormatterClass;
+ } else {
+ return formatter;
+ }
+ }
+
+ public void setLogErrorsToConsole() {
+ this.logErrorsToConsole = true;
+ }
+
+ private void logError(String s) {
+ log.error(s);
+ if (logErrorsToConsole) {
+ try {
+ reader.println("ERROR: " + s);
+ reader.flush();
+ } catch (IOException e) {}
+ }
+ }
+
+ public String readMaskedLine(String prompt, Character mask) throws IOException {
+ this.masking = true;
+ String s = reader.readLine(prompt, mask);
+ this.masking = false;
+ return s;
+ }
+
+ public boolean isMasking() {
+ return masking;
+ }
+
+ public boolean hasExited() {
+ return exit;
+ }
+
+ public boolean isTabCompletion() {
+ return tabCompletion;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
index d8b9f65,0000000..966fb00
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java
@@@ -1,60 -1,0 +1,61 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.File;
+import java.io.FileNotFoundException;
- import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Scanner;
+
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.collect.Lists;
+
+public class ShellUtil {
+
+ /**
+ * Scans the given file line-by-line (ignoring empty lines) and returns a list containing those lines. If decode is set to true, every line is decoded using
+ * {@link Base64#decodeBase64(byte[])} from the UTF-8 bytes of that line before inserting in the list.
+ *
+ * @param filename
+ * Path to the file that needs to be scanned
+ * @param decode
+ * Whether to decode lines in the file
+ * @return List of {@link Text} objects containing data in the given file
+ * @throws FileNotFoundException
+ * if the given file doesn't exist
+ */
+ public static List<Text> scanFile(String filename, boolean decode) throws FileNotFoundException {
+ String line;
- Scanner file = new Scanner(new File(filename), StandardCharsets.UTF_8.name());
++ Scanner file = new Scanner(new File(filename), UTF_8.name());
+ List<Text> result = Lists.newArrayList();
+ try {
+ while (file.hasNextLine()) {
+ line = file.nextLine();
+ if (!line.isEmpty()) {
- result.add(decode ? new Text(Base64.decodeBase64(line.getBytes(StandardCharsets.UTF_8))) : new Text(line));
++ result.add(decode ? new Text(Base64.decodeBase64(line.getBytes(UTF_8))) : new Text(line));
+ }
+ }
+ } finally {
+ file.close();
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
index 807de24,0000000..ef35f52
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/AuthenticateCommand.java
@@@ -1,66 -1,0 +1,67 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.Token;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.commons.cli.CommandLine;
+
+public class AuthenticateCommand extends Command {
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, IOException {
+ final String user = cl.getArgs()[0];
+ final String p = shellState.readMaskedLine("Enter current password for '" + user + "': ", '*');
+ if (p == null) {
+ shellState.getReader().println();
+ return 0;
+ } // user canceled
- final byte[] password = p.getBytes(StandardCharsets.UTF_8);
++ final byte[] password = p.getBytes(UTF_8);
+ final boolean valid = shellState.getConnector().securityOperations().authenticateUser(user, new PasswordToken(password));
+ shellState.getReader().println((valid ? "V" : "Not v") + "alid");
+ return 0;
+ }
+
+ @Override
+ public String description() {
+ return "verifies a user's credentials";
+ }
+
+ @Override
+ public String usage() {
+ return getName() + " <username>";
+ }
+
+ @Override
+ public void registerCompletion(final Token root, final Map<Command.CompletionSet,Set<String>> completionSet) {
+ registerCompletionForUsers(root, completionSet);
+ }
+
+ @Override
+ public int numArgs() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
index b384d5c,0000000..fb8af7f
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ExecfileCommand.java
@@@ -1,67 -1,0 +1,68 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.File;
- import java.nio.charset.StandardCharsets;
+import java.util.Scanner;
+
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+public class ExecfileCommand extends Command {
+ private Option verboseOption;
+
+ @Override
+ public String description() {
+ return "specifies a file containing accumulo commands to execute";
+ }
+
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
- Scanner scanner = new Scanner(new File(cl.getArgs()[0]), StandardCharsets.UTF_8.name());
++ Scanner scanner = new Scanner(new File(cl.getArgs()[0]), UTF_8.name());
+ try {
+ while (scanner.hasNextLine()) {
+ shellState.execCommand(scanner.nextLine(), true, cl.hasOption(verboseOption.getOpt()));
+ }
+ } finally {
+ scanner.close();
+ }
+ return 0;
+ }
+
+ @Override
+ public String usage() {
+ return getName() + " <fileName>";
+ }
+
+ @Override
+ public int numArgs() {
+ return 1;
+ }
+
+ @Override
+ public Options getOptions() {
+ final Options opts = new Options();
+ verboseOption = new Option("v", "verbose", false, "display command prompt as commands are executed");
+ opts.addOption(verboseOption);
+ return opts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
index 8cbf214,0000000..f21e33a
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/HiddenCommand.java
@@@ -1,62 -1,0 +1,63 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.security.SecureRandom;
+import java.util.Random;
+
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.ShellCommandException;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.accumulo.shell.ShellCommandException.ErrorCode;
+import org.apache.commons.cli.CommandLine;
+
+public class HiddenCommand extends Command {
+ private static Random rand = new SecureRandom();
+
+ @Override
+ public String description() {
+ return "The first rule of Accumulo is: \"You don't talk about Accumulo.\"";
+ }
+
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
+ if (rand.nextInt(10) == 0) {
+ shellState.getReader().beep();
+ shellState.getReader().println();
+ shellState.getReader().println(
+ new String(Base64.decodeBase64(("ICAgICAgIC4tLS4KICAgICAgLyAvXCBcCiAgICAgKCAvLS1cICkKICAgICAuPl8gIF88LgogICAgLyB8ICd8ICcgXAog"
+ + "ICAvICB8Xy58Xy4gIFwKICAvIC98ICAgICAgfFwgXAogfCB8IHwgfFwvfCB8IHwgfAogfF98IHwgfCAgfCB8IHxffAogICAgIC8gIF9fICBcCiAgICAvICAv"
- + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8));
++ + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(UTF_8)), UTF_8));
+ } else {
+ throw new ShellCommandException(ErrorCode.UNRECOGNIZED_COMMAND, getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public int numArgs() {
+ return Shell.NO_FIXED_ARG_LENGTH_CHECK;
+ }
+
+ @Override
+ public String getName() {
+ return "accvmvlo";
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
index 86c78fc,0000000..dd38889
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/PasswdCommand.java
@@@ -1,97 -1,0 +1,97 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
- import java.io.IOException;
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
++import java.io.IOException;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.accumulo.shell.ShellOptions;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+public class PasswdCommand extends Command {
+ private Option userOpt;
+
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, IOException {
+ final String currentUser = shellState.getConnector().whoami();
+ final String user = cl.getOptionValue(userOpt.getOpt(), currentUser);
+
+ String password = null;
+ String passwordConfirm = null;
+ String oldPassword = null;
+
+ oldPassword = shellState.readMaskedLine("Enter current password for '" + currentUser + "': ", '*');
+ if (oldPassword == null) {
+ shellState.getReader().println();
+ return 0;
+ } // user canceled
+
+ if (!shellState.getConnector().securityOperations().authenticateUser(currentUser, new PasswordToken(oldPassword)))
+ throw new AccumuloSecurityException(user, SecurityErrorCode.BAD_CREDENTIALS);
+
+ password = shellState.readMaskedLine("Enter new password for '" + user + "': ", '*');
+ if (password == null) {
+ shellState.getReader().println();
+ return 0;
+ } // user canceled
+ passwordConfirm = shellState.readMaskedLine("Please confirm new password for '" + user + "': ", '*');
+ if (passwordConfirm == null) {
+ shellState.getReader().println();
+ return 0;
+ } // user canceled
+
+ if (!password.equals(passwordConfirm)) {
+ throw new IllegalArgumentException("Passwords do not match");
+ }
- byte[] pass = password.getBytes(StandardCharsets.UTF_8);
++ byte[] pass = password.getBytes(UTF_8);
+ shellState.getConnector().securityOperations().changeLocalUserPassword(user, new PasswordToken(pass));
+ // update the current credentials if the password changed was for
+ // the current user
+ if (shellState.getConnector().whoami().equals(user)) {
+ shellState.updateUser(user, new PasswordToken(pass));
+ }
+ Shell.log.debug("Changed password for user " + user);
+ return 0;
+ }
+
+ @Override
+ public String description() {
+ return "changes a user's password";
+ }
+
+ @Override
+ public Options getOptions() {
+ final Options o = new Options();
+ userOpt = new Option(ShellOptions.userOption, "user", true, "user to operate on");
+ userOpt.setArgName("user");
+ o.addOption(userOpt);
+ return o;
+ }
+
+ @Override
+ public int numArgs() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
index ecc51eb,0000000..2fdb7fe
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/QuotedStringTokenizer.java
@@@ -1,141 -1,0 +1,142 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.UnsupportedEncodingException;
- import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.shell.Shell;
+
+/**
+ * A basic tokenizer for generating tokens from a string. It understands quoted strings and escaped quote characters.
+ *
+ * You can use the escape sequence '\' to escape single quotes, double quotes, and spaces only, in addition to the escape character itself.
+ *
+ * The behavior is the same for single and double quoted strings. (i.e. '\'' is the same as "\'")
+ */
+
+public class QuotedStringTokenizer implements Iterable<String> {
+ private ArrayList<String> tokens;
+ private String input;
+
+ public QuotedStringTokenizer(final String t) throws BadArgumentException {
+ tokens = new ArrayList<String>();
+ this.input = t;
+ try {
+ createTokens();
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+
+ public String[] getTokens() {
+ return tokens.toArray(new String[tokens.size()]);
+ }
+
+ private void createTokens() throws BadArgumentException, UnsupportedEncodingException {
+ boolean inQuote = false;
+ boolean inEscapeSequence = false;
+ String hexChars = null;
+ char inQuoteChar = '"';
+
+ final byte[] token = new byte[input.length()];
+ int tokenLength = 0;
- final byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
++ final byte[] inputBytes = input.getBytes(UTF_8);
+ for (int i = 0; i < input.length(); ++i) {
+ final char ch = input.charAt(i);
+
+ // if I ended up in an escape sequence, check for valid escapable character, and add it as a literal
+ if (inEscapeSequence) {
+ inEscapeSequence = false;
+ if (ch == 'x') {
+ hexChars = "";
+ } else if (ch == ' ' || ch == '\'' || ch == '"' || ch == '\\') {
+ token[tokenLength++] = inputBytes[i];
+ } else {
+ throw new BadArgumentException("can only escape single quotes, double quotes, the space character, the backslash, and hex input", input, i);
+ }
+ }
+ // in a hex escape sequence
+ else if (hexChars != null) {
+ final int digit = Character.digit(ch, 16);
+ if (digit < 0) {
+ throw new BadArgumentException("expected hex character", input, i);
+ }
+ hexChars += ch;
+ if (hexChars.length() == 2) {
+ byte b;
+ try {
+ b = (byte) (0xff & Short.parseShort(hexChars, 16));
+ if (!Character.isValidCodePoint(0xff & b))
+ throw new NumberFormatException();
+ } catch (NumberFormatException e) {
+ throw new BadArgumentException("unsupported non-ascii character", input, i);
+ }
+ token[tokenLength++] = b;
+ hexChars = null;
+ }
+ }
+ // in a quote, either end the quote, start escape, or continue a token
+ else if (inQuote) {
+ if (ch == inQuoteChar) {
+ inQuote = false;
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ tokenLength = 0;
+ } else if (ch == '\\') {
+ inEscapeSequence = true;
+ } else {
+ token[tokenLength++] = inputBytes[i];
+ }
+ }
+ // not in a quote, either enter a quote, end a token, start escape, or continue a token
+ else {
+ if (ch == '\'' || ch == '"') {
+ if (tokenLength > 0) {
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ tokenLength = 0;
+ }
+ inQuote = true;
+ inQuoteChar = ch;
+ } else if (ch == ' ' && tokenLength > 0) {
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ tokenLength = 0;
+ } else if (ch == '\\') {
+ inEscapeSequence = true;
+ } else if (ch != ' ') {
+ token[tokenLength++] = inputBytes[i];
+ }
+ }
+ }
+ if (inQuote) {
+ throw new BadArgumentException("missing terminating quote", input, input.length());
+ } else if (inEscapeSequence || hexChars != null) {
+ throw new BadArgumentException("escape sequence not complete", input, input.length());
+ }
+ if (tokenLength > 0) {
+ tokens.add(new String(token, 0, tokenLength, Shell.CHARSET));
+ }
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return tokens.iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
index 491f144,0000000..6e7247e
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/UserCommand.java
@@@ -1,71 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.Token;
+import org.apache.accumulo.shell.Shell.Command;
+import org.apache.commons.cli.CommandLine;
+
+public class UserCommand extends Command {
+ @Override
+ public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, IOException {
+ // save old credentials and connection in case of failure
+ String user = cl.getArgs()[0];
+ byte[] pass;
+
+ // We can't let the wrapping try around the execute method deal
+ // with the exceptions because we have to do something if one
+ // of these methods fails
+ final String p = shellState.readMaskedLine("Enter password for user " + user + ": ", '*');
+ if (p == null) {
+ shellState.getReader().println();
+ return 0;
+ } // user canceled
- pass = p.getBytes(StandardCharsets.UTF_8);
++ pass = p.getBytes(UTF_8);
+ shellState.updateUser(user, new PasswordToken(pass));
+ return 0;
+ }
+
+ @Override
+ public String description() {
+ return "switches to the specified user";
+ }
+
+ @Override
+ public void registerCompletion(final Token root, final Map<Command.CompletionSet,Set<String>> special) {
+ registerCompletionForUsers(root, special);
+ }
+
+ @Override
+ public String usage() {
+ return getName() + " <username>";
+ }
+
+ @Override
+ public int numArgs() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
----------------------------------------------------------------------
diff --cc shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
index cf028eb,0000000..f862b68
mode 100644,000000..100644
--- a/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/mock/MockShell.java
@@@ -1,143 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.mock;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
- import java.nio.charset.StandardCharsets;
-
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.shell.ShellOptionsJC;
+
+/**
+ * An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes.
+ */
+public class MockShell extends Shell {
+ private static final String NEWLINE = "\n";
+
+ protected InputStream in;
+ protected OutputStream out;
+
+ public MockShell(InputStream in, OutputStream out) throws IOException {
+ super();
+ this.in = in;
+ this.out = out;
+ }
+
+ public boolean config(String... args) {
+ configError = super.config(args);
+
+ // Update the ConsoleReader with the input and output "redirected"
+ try {
+ this.reader = new ConsoleReader(in, out);
+ } catch (Exception e) {
+ printException(e);
+ configError = true;
+ }
+
+ // Don't need this for testing purposes
+ this.reader.setHistoryEnabled(false);
+ this.reader.setPaginationEnabled(false);
+
+ // Make the parsing from the client easier;
+ this.verbose = false;
+ return configError;
+ }
+
+ @Override
+ protected void setInstance(ShellOptionsJC options) {
+ // We always want a MockInstance for this test
+ instance = new MockInstance();
+ }
+
+ public int start() throws IOException {
+ if (configError)
+ return 1;
+
+ String input;
+ if (isVerbose())
+ printInfo();
+
+ if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(execFile, StandardCharsets.UTF_8.name());
++ java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
+ try {
+ while (scanner.hasNextLine() && !hasExited()) {
+ execCommand(scanner.nextLine(), true, isVerbose());
+ }
+ } finally {
+ scanner.close();
+ }
+ } else if (execCommand != null) {
+ for (String command : execCommand.split("\n")) {
+ execCommand(command, true, isVerbose());
+ }
+ return exitCode;
+ }
+
+ while (true) {
+ if (hasExited())
+ return exitCode;
+
+ reader.setPrompt(getDefaultPrompt());
+ input = reader.readLine();
+ if (input == null) {
+ reader.println();
+ return exitCode;
+ } // user canceled
+
+ execCommand(input, false, false);
+ }
+ }
+
+ /**
+ * @param in
+ * the in to set
+ */
+ public void setConsoleInputStream(InputStream in) {
+ this.in = in;
+ }
+
+ /**
+ * @param out
+ * the output stream to set
+ */
+ public void setConsoleWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ /**
+ * Convenience method to create the byte-array to hand to the console
+ *
+ * @param commands
+ * An array of commands to run
+ * @return A byte[] input stream which can be handed to the console.
+ */
+ public static ByteArrayInputStream makeCommands(String... commands) {
+ StringBuilder sb = new StringBuilder(commands.length * 8);
+
+ for (String command : commands) {
+ sb.append(command).append(NEWLINE);
+ }
+
- return new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
++ return new ByteArrayInputStream(sb.toString().getBytes(UTF_8));
+ }
+}
[07/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index 665b132,0000000..4975428
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -1,356 -1,0 +1,357 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+
+import jline.console.ConsoleReader;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.shell.Shell;
+
+public class ShellServlet extends BasicServlet {
+ private static final long serialVersionUID = 1L;
+ private Map<String,ShellExecutionThread> userShells = new HashMap<String,ShellExecutionThread>();
+ private ExecutorService service = Executors.newCachedThreadPool();
+
+ public static final String CSRF_KEY = "csrf_token";
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "Shell";
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws IOException {
+ HttpSession session = req.getSession(true);
+ final String CSRF_TOKEN;
+ if (null == session.getAttribute(CSRF_KEY)) {
+ // No token, make one
+ CSRF_TOKEN = UUID.randomUUID().toString();
+ session.setAttribute(CSRF_KEY, CSRF_TOKEN);
+ } else {
+ // Pull the token out of the session
+ CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
+ if (null == CSRF_TOKEN) {
+ throw new RuntimeException("No valid CSRF token exists in session");
+ }
+ }
+
+ String user = (String) session.getAttribute("user");
+ if (user == null) {
+ // user attribute is null, check to see if username and password are passed as parameters
+ user = req.getParameter("user");
+ String pass = req.getParameter("pass");
+ String mock = req.getParameter("mock");
+ if (user == null || pass == null) {
+ // username or password are null, re-authenticate
+ sb.append(authenticationForm(req.getRequestURI(), CSRF_TOKEN));
+ return;
+ }
+ try {
+ // get a new shell for this user
+ ShellExecutionThread shellThread = new ShellExecutionThread(user, pass, mock);
+ service.submit(shellThread);
+ userShells.put(session.getId(), shellThread);
+ } catch (IOException e) {
+ // error validating user, reauthenticate
+ sb.append("<div id='loginError'>Invalid user/password</div>" + authenticationForm(req.getRequestURI(), CSRF_TOKEN));
+ return;
+ }
+ session.setAttribute("user", user);
+ }
+ if (!userShells.containsKey(session.getId())) {
+ // no existing shell for this user, re-authenticate
+ sb.append(authenticationForm(req.getRequestURI(), UUID.randomUUID().toString()));
+ return;
+ }
+
+ ShellExecutionThread shellThread = userShells.get(session.getId());
+ shellThread.getOutput();
+ shellThread.printInfo();
+ sb.append("<div id='shell'>\n");
+ sb.append("<pre id='shellResponse'>").append(shellThread.getOutput()).append("</pre>\n");
+ sb.append("<form><span id='shellPrompt'>").append(shellThread.getPrompt());
+ sb.append("</span><input type='text' name='cmd' id='cmd' onkeydown='return handleKeyDown(event.keyCode);'>\n");
+ sb.append("</form>\n</div>\n");
+ sb.append("<script type='text/javascript'>\n");
+ sb.append("var url = '").append(req.getRequestURL().toString()).append("';\n");
+ sb.append("var xmlhttp = new XMLHttpRequest();\n");
+ sb.append("var hsize = 1000;\n");
+ sb.append("var hindex = 0;\n");
+ sb.append("var history = new Array();\n");
+ sb.append("\n");
+ sb.append("function handleKeyDown(keyCode) {\n");
+ sb.append(" if (keyCode==13) {\n");
+ sb.append(" submitCmd(document.getElementById('cmd').value);\n");
+ sb.append(" hindex = history.length;\n");
+ sb.append(" return false;\n");
+ sb.append(" } else if (keyCode==38) {\n");
+ sb.append(" hindex = hindex==0 ? history.length : hindex - 1;\n");
+ sb.append(" if (hindex == history.length)\n");
+ sb.append(" document.getElementById('cmd').value = '';\n");
+ sb.append(" else\n");
+ sb.append(" document.getElementById('cmd').value = history[hindex];\n");
+ sb.append(" return false;\n");
+ sb.append(" } else if (keyCode==40) {\n");
+ sb.append(" hindex = hindex==history.length ? history.length : hindex + 1;\n");
+ sb.append(" if (hindex == history.length)\n");
+ sb.append(" document.getElementById('cmd').value = '';\n");
+ sb.append(" else\n");
+ sb.append(" document.getElementById('cmd').value = history[hindex];\n");
+ sb.append(" return false;\n");
+ sb.append(" }\n");
+ sb.append(" return true;\n");
+ sb.append("}\n");
+ sb.append("\n");
+ sb.append("function submitCmd(cmd) {\n");
+ sb.append(" if (cmd=='history') {\n");
+ sb.append(" document.getElementById('shellResponse').innerHTML += document.getElementById('shellPrompt').innerHTML+cmd+'\\n';\n");
+ sb.append(" document.getElementById('shellResponse').innerHTML += history.join('\\n');\n");
+ sb.append(" return\n");
+ sb.append(" }\n");
+ sb.append(" xmlhttp.open('POST',url+'?cmd='+cmd+'&'+'").append(CSRF_KEY).append("=").append(CSRF_TOKEN).append("',false);\n");
+ sb.append(" xmlhttp.send();\n");
+ sb.append(" var text = xmlhttp.responseText;\n");
+ sb.append(" var index = text.lastIndexOf('\\n');\n");
+ sb.append(" if (index >= 0) {\n");
+ sb.append(" if (index > 0 && document.getElementById('cmd').type == 'text') {\n");
+ sb.append(" if (history.length == hsize)\n");
+ sb.append(" history.shift()\n");
+ sb.append(" history.push(cmd)\n");
+ sb.append(" }\n");
+ sb.append(" if (text.charAt(text.length-1)=='*') {\n");
+ sb.append(" document.getElementById('cmd').type = 'password';\n");
+ sb.append(" text = text.substring(0,xmlhttp.responseText.length-2);\n");
+ sb.append(" } else {\n");
+ sb.append(" document.getElementById('cmd').type = 'text';\n");
+ sb.append(" }\n");
+ sb.append(" document.getElementById('shellResponse').innerHTML += text.substring(0,index+1);\n");
+ sb.append(" document.getElementById('shellPrompt').innerHTML = text.substring(index+1);\n");
+ sb.append(" document.getElementById('cmd').value = '';\n");
+ sb.append(" document.getElementById('shell').scrollTop = document.getElementById('cmd').offsetTop;\n");
+ sb.append(" } else {\n");
+ sb.append(" window.location = url;\n");
+ sb.append(" }\n");
+ sb.append("}\n");
+ sb.append("</script>\n");
+ sb.append("<script type='text/javascript'>window.onload = function() { document.getElementById('cmd').select(); }</script>\n");
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ final HttpSession session = req.getSession(true);
+ String user = (String) session.getAttribute("user");
+ if (user == null || !userShells.containsKey(session.getId())) {
+ // no existing shell for user, re-authenticate
+ doGet(req, resp);
+ return;
+ }
+ final String CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
+ if (null == CSRF_TOKEN) {
+ // no csrf token, need to re-auth
+ doGet(req, resp);
+ }
+ ShellExecutionThread shellThread = userShells.get(session.getId());
+ String cmd = req.getParameter("cmd");
+ if (cmd == null) {
+ // the command is null, just print prompt
+ resp.getWriter().append(shellThread.getPrompt());
+ resp.getWriter().flush();
+ return;
+ }
+ shellThread.addInputString(cmd);
+ shellThread.waitUntilReady();
+ if (shellThread.isDone()) {
+ // the command was exit, invalidate session
+ userShells.remove(session.getId());
+ session.invalidate();
+ return;
+ }
+ // get the shell's output
+ StringBuilder sb = new StringBuilder();
+ sb.append(shellThread.getOutput().replace("<", "<").replace(">", ">"));
+ if (sb.length() == 0 || !(sb.charAt(sb.length() - 1) == '\n'))
+ sb.append("\n");
+ // check if shell is waiting for input
+ if (!shellThread.isWaitingForInput())
+ sb.append(shellThread.getPrompt());
+ // check if shell is waiting for password input
+ if (shellThread.isMasking())
+ sb.append("*");
+ resp.getWriter().append(sb.toString());
+ resp.getWriter().flush();
+ }
+
+ private String authenticationForm(String requestURI, String csrfToken) {
+ return "<div id='login'><form method=POST action='" + requestURI + "'>"
+ + "<table><tr><td>Mock: </td><td><input type='checkbox' name='mock' value='mock'></td></tr>"
+ + "<tr><td>Username: </td><td><input type='text' name='user'></td></tr>"
+ + "<tr><td>Password: </td><td><input type='password' name='pass'></td><td>"
+ + "<input type='hidden' name='" + CSRF_KEY + "' value='" + csrfToken + "'/><input type='submit' value='Enter'></td></tr></table></form></div>";
+ }
+
+ private static class StringBuilderOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static class ShellExecutionThread extends InputStream implements Runnable {
+ private Shell shell;
+ StringBuilderOutputStream output;
+ private String cmd;
+ private int cmdIndex;
+ private boolean done;
+ private boolean readWait;
+
+ private ShellExecutionThread(String username, String password, String mock) throws IOException {
+ this.done = false;
+ this.cmd = null;
+ this.cmdIndex = 0;
+ this.readWait = false;
+ this.output = new StringBuilderOutputStream();
+ ConsoleReader reader = new ConsoleReader(this, output);
- this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, Constants.UTF8)));
++ this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, UTF_8)));
+ shell.setLogErrorsToConsole();
+ if (mock != null) {
+ if (shell.config("--fake", "-u", username, "-p", password))
+ throw new IOException("mock shell config error");
+ } else if (shell.config("-u", username, "-p", password)) {
+ throw new IOException("shell config error");
+ }
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ if (cmd == null) {
+ readWait = true;
+ this.notifyAll();
+ }
+ while (cmd == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {}
+ }
+ readWait = false;
+ int c;
+ if (cmdIndex == cmd.length())
+ c = '\n';
+ else
+ c = cmd.charAt(cmdIndex);
+ cmdIndex++;
+ if (cmdIndex > cmd.length()) {
+ cmd = null;
+ cmdIndex = 0;
+ this.notifyAll();
+ }
+ return c;
+ }
+
+ @Override
+ public synchronized void run() {
+ Thread.currentThread().setName("shell thread");
+ while (!shell.hasExited()) {
+ while (cmd == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {}
+ }
+ String tcmd = cmd;
+ cmd = null;
+ cmdIndex = 0;
+ try {
+ shell.execCommand(tcmd, false, true);
+ } catch (IOException e) {}
+ this.notifyAll();
+ }
+ done = true;
+ this.notifyAll();
+ }
+
+ public synchronized void addInputString(String s) {
+ if (done)
+ throw new IllegalStateException("adding string to exited shell");
+ if (cmd == null) {
+ cmd = s;
+ } else {
+ throw new IllegalStateException("adding string to shell not waiting for input");
+ }
+ this.notifyAll();
+ }
+
+ public synchronized void waitUntilReady() {
+ while (cmd != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+
+ public synchronized String getOutput() {
+ String s = output.get();
+ output.clear();
+ return s;
+ }
+
+ public String getPrompt() {
+ return shell.getDefaultPrompt();
+ }
+
+ public void printInfo() throws IOException {
+ shell.printInfo();
+ }
+
+ public boolean isMasking() {
+ return shell.isMasking();
+ }
+
+ public synchronized boolean isWaitingForInput() {
+ return readWait;
+ }
+
+ public boolean isDone() {
+ return done;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
index 0cbf957,0000000..3cd635e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
@@@ -1,179 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.DeadServer;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.state.TabletServerState;
+import org.apache.accumulo.server.util.TableInfoUtil;
+
+public class XMLServlet extends BasicServlet {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "XML Report";
+ }
+
+ @Override
+ protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
- resp.setContentType("text/xml;charset=" + Constants.UTF8.name());
- sb.append("<?xml version=\"1.0\" encoding=\"" + Constants.UTF8.name() + "\"?>\n");
++ resp.setContentType("text/xml;charset=" + UTF_8.name());
++ sb.append("<?xml version=\"1.0\" encoding=\"" + UTF_8.name() + "\"?>\n");
+ sb.append("<stats>\n");
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
+ double totalIngest = 0.;
+ double totalQuery = 0.;
+ double disk = 0.0;
+ long totalEntries = 0L;
+
+ sb.append("\n<servers>\n");
+ if (Monitor.getMmi() == null || Monitor.getMmi().tableMap == null) {
+ sb.append("</servers>\n");
+ return;
+ }
+ SortedMap<String,TableInfo> tableStats = new TreeMap<String,TableInfo>(Monitor.getMmi().tableMap);
+
+ for (TabletServerStatus status : Monitor.getMmi().tServerInfo) {
+
+ sb.append("\n<server id='").append(status.name).append("'>\n");
+ sb.append("<hostname>").append(TServerLinkType.displayName(status.name)).append("</hostname>");
+ sb.append("<lastContact>").append(System.currentTimeMillis() - status.lastContact).append("</lastContact>\n");
+ sb.append("<osload>").append(status.osLoad).append("</osload>\n");
+
+ TableInfo summary = TableInfoUtil.summarizeTableStats(status);
+ sb.append("<compactions>\n");
+ sb.append("<major>").append("<running>").append(summary.majors.running).append("</running>").append("<queued>").append(summary.majors.queued)
+ .append("</queued>").append("</major>\n");
+ sb.append("<minor>").append("<running>").append(summary.minors.running).append("</running>").append("<queued>").append(summary.minors.queued)
+ .append("</queued>").append("</minor>\n");
+ sb.append("</compactions>\n");
+
+ sb.append("<tablets>").append(summary.tablets).append("</tablets>\n");
+
+ sb.append("<ingest>").append(summary.ingestRate).append("</ingest>\n");
+ sb.append("<query>").append(summary.queryRate).append("</query>\n");
+ sb.append("<ingestMB>").append(summary.ingestByteRate / 1000000.0).append("</ingestMB>\n");
+ sb.append("<queryMB>").append(summary.queryByteRate / 1000000.0).append("</queryMB>\n");
+ sb.append("<scans>").append(summary.scans.running + summary.scans.queued).append("</scans>");
+ sb.append("<scansessions>").append(Monitor.getLookupRate()).append("</scansessions>\n");
+ sb.append("<holdtime>").append(status.holdTime).append("</holdtime>\n");
+ totalIngest += summary.ingestRate;
+ totalQuery += summary.queryRate;
+ totalEntries += summary.recs;
+ sb.append("</server>\n");
+ }
+ sb.append("\n</servers>\n");
+
+ sb.append("\n<masterGoalState>" + Monitor.getMmi().goalState + "</masterGoalState>\n");
+ sb.append("\n<masterState>" + Monitor.getMmi().state + "</masterState>\n");
+
+ sb.append("\n<badTabletServers>\n");
+ for (Entry<String,Byte> entry : Monitor.getMmi().badTServers.entrySet()) {
+ sb.append(String.format("<badTabletServer id='%s' status='%s'/>\n", entry.getKey(), TabletServerState.getStateById(entry.getValue())));
+ }
+ sb.append("\n</badTabletServers>\n");
+
+ sb.append("\n<tabletServersShuttingDown>\n");
+ for (String server : Monitor.getMmi().serversShuttingDown) {
+ sb.append(String.format("<server id='%s'/>\n", server));
+ }
+ sb.append("\n</tabletServersShuttingDown>\n");
+
+ sb.append(String.format("\n<unassignedTablets>%d</unassignedTablets>\n", Monitor.getMmi().unassignedTablets));
+
+ sb.append("\n<deadTabletServers>\n");
+ for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
+ sb.append(String.format("<deadTabletServer id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
+ }
+ sb.append("\n</deadTabletServers>\n");
+
+ sb.append("\n<deadLoggers>\n");
+ for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
+ sb.append(String.format("<deadLogger id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
+ }
+ sb.append("\n</deadLoggers>\n");
+
+ sb.append("\n<tables>\n");
+ Instance instance = HdfsZooInstance.getInstance();
+ for (Entry<String,TableInfo> entry : tableStats.entrySet()) {
+ TableInfo tableInfo = entry.getValue();
+
+ sb.append("\n<table>\n");
+ String tableId = entry.getKey();
+ String tableName = "unknown";
+ String tableState = "unknown";
+ try {
+ tableName = Tables.getTableName(instance, tableId);
+ tableState = Tables.getTableState(instance, tableId).toString();
+ } catch (Exception ex) {
+ log.warn(ex, ex);
+ }
+ sb.append("<tablename>").append(tableName).append("</tablename>\n");
+ sb.append("<tableId>").append(tableId).append("</tableId>\n");
+ sb.append("<tableState>").append(tableState).append("</tableState>\n");
+ sb.append("<tablets>").append(tableInfo.tablets).append("</tablets>\n");
+ sb.append("<onlineTablets>").append(tableInfo.onlineTablets).append("</onlineTablets>\n");
+ sb.append("<recs>").append(tableInfo.recs).append("</recs>\n");
+ sb.append("<recsInMemory>").append(tableInfo.recsInMemory).append("</recsInMemory>\n");
+ sb.append("<ingest>").append(tableInfo.ingestRate).append("</ingest>\n");
+ sb.append("<ingestByteRate>").append(tableInfo.ingestByteRate).append("</ingestByteRate>\n");
+ sb.append("<query>").append(tableInfo.queryRate).append("</query>\n");
+ sb.append("<queryByteRate>").append(tableInfo.queryRate).append("</queryByteRate>\n");
+ int running = 0;
+ int queued = 0;
+ Compacting compacting = entry.getValue().majors;
+ if (compacting != null) {
+ running = compacting.running;
+ queued = compacting.queued;
+ }
+ sb.append("<majorCompactions>").append("<running>").append(running).append("</running>").append("<queued>").append(queued).append("</queued>")
+ .append("</majorCompactions>\n");
+ sb.append("</table>\n");
+ }
+ sb.append("\n</tables>\n");
+
+ sb.append("\n<totals>\n");
+ sb.append("<ingestrate>").append(totalIngest).append("</ingestrate>\n");
+ sb.append("<queryrate>").append(totalQuery).append("</queryrate>\n");
+ sb.append("<diskrate>").append(disk).append("</diskrate>\n");
+ sb.append("<numentries>").append(totalEntries).append("</numentries>\n");
+ sb.append("</totals>\n");
+ }
+
+ @Override
+ protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
+ sb.append("\n</stats>\n");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 90bb0d4,0000000..1c464d4
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@@ -1,105 -1,0 +1,106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets.trace;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Date;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.servlet.http.HttpServletRequest;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.servlets.BasicServlet;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+abstract class Basic extends BasicServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ public static String getStringParameter(HttpServletRequest req, String name, String defaultValue) {
+ String result = req.getParameter(name);
+ if (result == null) {
+ return defaultValue;
+ }
+ return result;
+ }
+
+ public static int getIntParameter(HttpServletRequest req, String name, int defaultMinutes) {
+ String valueString = req.getParameter(name);
+ if (valueString == null)
+ return defaultMinutes;
+ int result = 0;
+ try {
+ result = Integer.parseInt(valueString);
+ } catch (NumberFormatException ex) {
+ return defaultMinutes;
+ }
+ return result;
+ }
+
+ public static String dateString(long millis) {
+ return TraceFormatter.formatDate(new Date(millis));
+ }
+
+ protected Scanner getScanner(StringBuilder sb) throws AccumuloException, AccumuloSecurityException {
+ AccumuloConfiguration conf = Monitor.getSystemConfiguration();
+ String principal = conf.get(Property.TRACE_USER);
+ AuthenticationToken at;
+ Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+ if (loginMap.isEmpty()) {
+ Property p = Property.TRACE_PASSWORD;
- at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++ at = new PasswordToken(conf.get(p).getBytes(UTF_8));
+ } else {
+ Properties props = new Properties();
+ int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
+ for (Entry<String,String> entry : loginMap.entrySet()) {
+ props.put(entry.getKey().substring(prefixLength), entry.getValue());
+ }
+
+ AuthenticationToken token = Property.createInstanceFromPropertyName(conf, Property.TRACE_TOKEN_TYPE, AuthenticationToken.class, new PasswordToken());
+ token.init(props);
+ at = token;
+ }
+
+ String table = conf.get(Property.TRACE_TABLE);
+ try {
+ Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
+ if (!conn.tableOperations().exists(table)) {
+ return new NullScanner();
+ }
+ Scanner scanner = conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
+ return scanner;
+ } catch (AccumuloSecurityException ex) {
+ sb.append("<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
+ return null;
+ } catch (TableNotFoundException ex) {
+ return new NullScanner();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
----------------------------------------------------------------------
diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index e3ae0c8,0000000..6094911
mode 100644,000000..100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -1,324 -1,0 +1,326 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tracer;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.thrift.RemoteSpan;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TByteArrayOutputStream;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+
+public class TraceServer implements Watcher {
+
+ final private static Logger log = Logger.getLogger(TraceServer.class);
+ final private ServerConfiguration serverConfiguration;
+ final private TServer server;
+ final private AtomicReference<BatchWriter> writer;
+ final private Connector connector;
+ final String table;
+
+ private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) {
+ m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
+ }
+
+ static class ByteArrayTransport extends TTransport {
+ TByteArrayOutputStream out = new TByteArrayOutputStream();
+
+ @Override
+ public boolean isOpen() {
+ return true;
+ }
+
+ @Override
+ public void open() throws TTransportException {}
+
+ @Override
+ public void close() {}
+
+ @Override
+ public int read(byte[] buf, int off, int len) {
+ return 0;
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) throws TTransportException {
+ out.write(buf, off, len);
+ }
+
+ public byte[] get() {
+ return out.get();
+ }
+
+ public int len() {
+ return out.len();
+ }
+ }
+
+ class Receiver implements Iface {
+ @Override
+ public void span(RemoteSpan s) throws TException {
+ String idString = Long.toHexString(s.traceId);
+ String startString = Long.toHexString(s.start);
+ Mutation spanMutation = new Mutation(new Text(idString));
+ Mutation indexMutation = new Mutation(new Text("idx:" + s.svc + ":" + startString));
+ long diff = s.stop - s.start;
- indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(Constants.UTF8)));
++ indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(UTF_8)));
+ ByteArrayTransport transport = new ByteArrayTransport();
+ TCompactProtocol protocol = new TCompactProtocol(transport);
+ s.write(protocol);
+ String parentString = Long.toHexString(s.parentId);
+ if (s.parentId == Span.ROOT_SPAN_ID)
+ parentString = "";
+ put(spanMutation, "span", parentString + ":" + Long.toHexString(s.spanId), transport.get(), transport.len());
+ // Map the root span to time so we can look up traces by time
+ Mutation timeMutation = null;
+ if (s.parentId == Span.ROOT_SPAN_ID) {
+ timeMutation = new Mutation(new Text("start:" + startString));
+ put(timeMutation, "id", idString, transport.get(), transport.len());
+ }
+ try {
+ final BatchWriter writer = TraceServer.this.writer.get();
+ /*
+ * Check for null, because we expect spans to come in much faster than flush calls. In the case of failure, we'd rather avoid logging tons of NPEs.
+ */
+ if (null == writer) {
+ log.warn("writer is not ready; discarding span.");
+ return;
+ }
+ writer.addMutation(spanMutation);
+ writer.addMutation(indexMutation);
+ if (timeMutation != null)
+ writer.addMutation(timeMutation);
+ } catch (MutationsRejectedException exception) {
+ log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception);
+ if (log.isDebugEnabled()) {
+ log.debug("discarded span due to rejection of mutation: " + spanMutation, exception);
+ }
+ /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */
+ } catch (RuntimeException exception) {
+ log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception);
+ log.debug("unable to write mutation to table due to exception.", exception);
+ }
+ }
+
+ }
+
+ public TraceServer(ServerConfiguration serverConfiguration, String hostname) throws Exception {
+ this.serverConfiguration = serverConfiguration;
+ AccumuloConfiguration conf = serverConfiguration.getConfiguration();
+ table = conf.get(Property.TRACE_TABLE);
+ Connector connector = null;
+ while (true) {
+ try {
+ String principal = conf.get(Property.TRACE_USER);
+ AuthenticationToken at;
+ Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+ if (loginMap.isEmpty()) {
+ Property p = Property.TRACE_PASSWORD;
- at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++ at = new PasswordToken(conf.get(p).getBytes(UTF_8));
+ } else {
+ Properties props = new Properties();
+ AuthenticationToken token = AccumuloVFSClassLoader.getClassLoader().loadClass(conf.get(Property.TRACE_TOKEN_TYPE)).asSubclass(AuthenticationToken.class)
+ .newInstance();
+
+ int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
+ for (Entry<String,String> entry : loginMap.entrySet()) {
+ props.put(entry.getKey().substring(prefixLength), entry.getValue());
+ }
+
+ token.init(props);
+
+ at = token;
+ }
+
+ connector = serverConfiguration.getInstance().getConnector(principal, at);
+ if (!connector.tableOperations().exists(table)) {
+ connector.tableOperations().create(table);
+ IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName());
+ AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
+ connector.tableOperations().attachIterator(table, setting);
+ }
+ connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
+ break;
+ } catch (Exception ex) {
+ log.info("Waiting to checking/create the trace table.", ex);
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ this.connector = connector;
+ // make sure we refer to the final variable from now on.
+ connector = null;
+
+ int port = conf.getPort(Property.TRACE_PORT);
+ final ServerSocket sock = ServerSocketChannel.open().socket();
+ sock.setReuseAddress(true);
+ sock.bind(new InetSocketAddress(hostname, port));
+ final TServerTransport transport = new TServerSocket(sock);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.processor(new Processor<Iface>(new Receiver()));
+ server = new TThreadPoolServer(options);
+ registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort());
+ writer = new AtomicReference<BatchWriter>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS)));
+ }
+
+ public void run() throws Exception {
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ flush();
+ }
+ }, 1000, 1000);
+ server.serve();
+ }
+
+ private void flush() {
+ try {
+ final BatchWriter writer = this.writer.get();
+ if (null != writer) {
+ writer.flush();
+ }
+ } catch (MutationsRejectedException exception) {
+ log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
+ log.debug("flushing traces failed due to exception", exception);
+ resetWriter();
+ /* XXX e.g. if the writer was closed between when we grabbed it and when we called flush. */
+ } catch (RuntimeException exception) {
+ log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
+ log.debug("flushing traces failed due to exception", exception);
+ resetWriter();
+ }
+ }
+
+ private void resetWriter() {
+ BatchWriter writer = null;
+ try {
+ writer = connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS));
+ } catch (Exception ex) {
+ log.warn("Unable to create a batch writer, will retry. Set log level to DEBUG to see stacktrace. cause: " + ex);
+ log.debug("batch writer creation failed with exception.", ex);
+ } finally {
+ /* Trade in the new writer (even if null) for the one we need to close. */
+ writer = this.writer.getAndSet(writer);
+ try {
+ if (null != writer) {
+ writer.close();
+ }
+ } catch (Exception ex) {
+ log.warn("Problem closing batch writer. Set log level to DEBUG to see stacktrace. cause: " + ex);
+ log.debug("batch writer close failed with exception", ex);
+ }
+ }
+ }
+
+ private void registerInZooKeeper(String name) throws Exception {
+ String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(Constants.UTF8));
++ String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(UTF_8));
+ zoo.exists(path, this);
+ }
+
+ public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+ ServerOpts opts = new ServerOpts();
+ final String app = "tracer";
+ opts.parseArgs(app, args);
+ Accumulo.setupLogging(app);
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ VolumeManager fs = VolumeManagerImpl.get();
+ Accumulo.init(fs, conf, app);
+ String hostname = opts.getAddress();
+ TraceServer server = new TraceServer(conf, hostname);
+ Accumulo.enableTracing(hostname, app);
+ server.run();
+ log.info("tracer stopping");
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
+ if (event.getState() == KeeperState.Expired) {
+ log.warn("Trace server lost zookeeper registration at " + event.getPath());
+ server.stop();
+ } else if (event.getType() == EventType.NodeDeleted) {
+ log.warn("Trace server zookeeper entry lost " + event.getPath());
+ server.stop();
+ }
+ if (event.getPath() != null) {
+ try {
+ if (ZooReaderWriter.getInstance().exists(event.getPath(), this))
+ return;
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ log.warn("Trace server unable to reset watch on zookeeper registration");
+ server.stop();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index 39bf81e,0000000..4d9a6c2
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@@ -1,75 -1,0 +1,76 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Copy failed bulk imports.
+ */
+public class BulkFailedCopyProcessor implements Processor {
+
+ private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
+
+ @Override
+ public Processor newProcessor() {
+ return new BulkFailedCopyProcessor();
+ }
+
+ @Override
+ public void process(String workID, byte[] data) {
+
- String paths[] = new String(data, Constants.UTF8).split(",");
++ String paths[] = new String(data, UTF_8).split(",");
+
+ Path orig = new Path(paths[0]);
+ Path dest = new Path(paths[1]);
+ Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
+
+ try {
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
+ FileSystem origFs = TraceFileSystem.wrap(vm.getVolumeByPath(orig).getFileSystem());
+ FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
+
+ FileUtil.copy(origFs, orig, destFs, tmp, false, true, CachedConfiguration.getInstance());
+ destFs.rename(tmp, dest);
+ log.debug("copied " + orig + " to " + dest);
+ } catch (IOException ex) {
+ try {
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
+ FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
+ destFs.create(dest).close();
+ log.warn(" marked " + dest + " failed", ex);
+ } catch (IOException e) {
+ log.error("Unable to create failure flag file " + dest, e);
+ }
+ }
+
+ }
+
+}
[25/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 0e02f05,0000000..7158ea8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@@ -1,598 -1,0 +1,598 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.log;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_FINISH;
+import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_START;
+import static org.apache.accumulo.tserver.logger.LogEvents.DEFINE_TABLET;
+import static org.apache.accumulo.tserver.logger.LogEvents.MANY_MUTATIONS;
+import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.crypto.CryptoModule;
+import org.apache.accumulo.core.security.crypto.CryptoModuleFactory;
+import org.apache.accumulo.core.security.crypto.CryptoModuleParameters;
+import org.apache.accumulo.core.security.crypto.DefaultCryptoModule;
+import org.apache.accumulo.core.security.crypto.NoFlushOutputStream;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.tserver.TabletMutations;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Wrap a connection to a logger.
+ *
+ */
+public class DfsLogger {
+ public static final String LOG_FILE_HEADER_V2 = "--- Log File Header (v2) ---";
+ public static final String LOG_FILE_HEADER_V3 = "--- Log File Header (v3) ---";
+
+ private static Logger log = Logger.getLogger(DfsLogger.class);
+
+ public static class LogClosedException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public LogClosedException() {
+ super("LogClosed");
+ }
+ }
+
+ /**
+ * A well-timed tabletserver failure could result in an incomplete header written to a write-ahead log. This exception is thrown when the header cannot be
+ * read from a WAL which should only happen when the tserver dies as described.
+ */
+ public static class LogHeaderIncompleteException extends IOException {
+ private static final long serialVersionUID = 1l;
+
+ public LogHeaderIncompleteException(String msg) {
+ super(msg);
+ }
+
+ public LogHeaderIncompleteException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+ public LogHeaderIncompleteException(Throwable cause) {
+ super(cause);
+ }
+ }
+
+ public static class DFSLoggerInputStreams {
+
+ private FSDataInputStream originalInput;
+ private DataInputStream decryptingInputStream;
+
+ public DFSLoggerInputStreams(FSDataInputStream originalInput, DataInputStream decryptingInputStream) {
+ this.originalInput = originalInput;
+ this.decryptingInputStream = decryptingInputStream;
+ }
+
+ public FSDataInputStream getOriginalInput() {
+ return originalInput;
+ }
+
+ public void setOriginalInput(FSDataInputStream originalInput) {
+ this.originalInput = originalInput;
+ }
+
+ public DataInputStream getDecryptingInputStream() {
+ return decryptingInputStream;
+ }
+
+ public void setDecryptingInputStream(DataInputStream decryptingInputStream) {
+ this.decryptingInputStream = decryptingInputStream;
+ }
+ }
+
+ public interface ServerResources {
+ AccumuloConfiguration getConfiguration();
+
+ VolumeManager getFileSystem();
+
+ Set<TServerInstance> getCurrentTServers();
+ }
+
+ private final LinkedBlockingQueue<DfsLogger.LogWork> workQueue = new LinkedBlockingQueue<DfsLogger.LogWork>();
+
+ private final Object closeLock = new Object();
+
+ private static final DfsLogger.LogWork CLOSED_MARKER = new DfsLogger.LogWork(null);
+
+ private static final LogFileValue EMPTY = new LogFileValue();
+
+ private boolean closed = false;
+
+ private class LogSyncingTask implements Runnable {
+
+ @Override
+ public void run() {
+ ArrayList<DfsLogger.LogWork> work = new ArrayList<DfsLogger.LogWork>();
+ boolean sawClosedMarker = false;
+ while (!sawClosedMarker) {
+ work.clear();
+
+ try {
+ work.add(workQueue.take());
+ } catch (InterruptedException ex) {
+ continue;
+ }
+ workQueue.drainTo(work);
+
+ try {
+ sync.invoke(logFile);
+ } catch (Exception ex) {
+ log.warn("Exception syncing " + ex);
+ for (DfsLogger.LogWork logWork : work) {
+ logWork.exception = ex;
+ }
+ }
+
+ for (DfsLogger.LogWork logWork : work)
+ if (logWork == CLOSED_MARKER)
+ sawClosedMarker = true;
+ else
+ logWork.latch.countDown();
+ }
+ }
+ }
+
+ static class LogWork {
+ CountDownLatch latch;
+ volatile Exception exception;
+
+ public LogWork(CountDownLatch latch) {
+ this.latch = latch;
+ }
+ }
+
+ public static class LoggerOperation {
+ private final LogWork work;
+
+ public LoggerOperation(LogWork work) {
+ this.work = work;
+ }
+
+ public void await() throws IOException {
+ try {
+ work.latch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ if (work.exception != null) {
+ if (work.exception instanceof IOException)
+ throw (IOException) work.exception;
+ else if (work.exception instanceof RuntimeException)
+ throw (RuntimeException) work.exception;
+ else
+ throw new RuntimeException(work.exception);
+ }
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // filename is unique
+ if (obj == null)
+ return false;
+ if (obj instanceof DfsLogger)
+ return getFileName().equals(((DfsLogger) obj).getFileName());
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ // filename is unique
+ return getFileName().hashCode();
+ }
+
+ private final ServerResources conf;
+ private FSDataOutputStream logFile;
+ private DataOutputStream encryptingLogFile = null;
+ private Method sync;
+ private String logPath;
+ private Daemon syncThread;
+
+ /* Track what's actually in +r/!0 for this logger ref */
+ private String metaReference;
+
+ public DfsLogger(ServerResources conf) throws IOException {
+ this.conf = conf;
+ }
+
+ /**
+ * Refernce a pre-existing log file.
+ *
+ * @param meta
+ * the cq for the "log" entry in +r/!0
+ */
+ public DfsLogger(ServerResources conf, String filename, String meta) throws IOException {
+ this.conf = conf;
+ this.logPath = filename;
+ metaReference = meta;
+ }
+
+ public static DFSLoggerInputStreams readHeaderAndReturnStream(VolumeManager fs, Path path, AccumuloConfiguration conf) throws IOException {
+ FSDataInputStream input = fs.open(path);
+ DataInputStream decryptingInput = null;
+
+ byte[] magic = DfsLogger.LOG_FILE_HEADER_V3.getBytes();
+ byte[] magicBuffer = new byte[magic.length];
+ try {
+ input.readFully(magicBuffer);
+ if (Arrays.equals(magicBuffer, magic)) {
+ // additional parameters it needs from the underlying stream.
+ String cryptoModuleClassname = input.readUTF();
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoModuleClassname);
+
+ // Create the parameters and set the input stream into those parameters
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setEncryptedInputStream(input);
+
+ // Create the plaintext input stream from the encrypted one
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ if (params.getPlaintextInputStream() instanceof DataInputStream) {
+ decryptingInput = (DataInputStream) params.getPlaintextInputStream();
+ } else {
+ decryptingInput = new DataInputStream(params.getPlaintextInputStream());
+ }
+ } else {
+ input.seek(0);
+ byte[] magicV2 = DfsLogger.LOG_FILE_HEADER_V2.getBytes();
+ byte[] magicBufferV2 = new byte[magicV2.length];
+ input.readFully(magicBufferV2);
+
+ if (Arrays.equals(magicBufferV2, magicV2)) {
+ // Log files from 1.5 dump their options in raw to the logger files. Since we don't know the class
+ // that needs to read those files, we can make a couple of basic assumptions. Either it's going to be
+ // the NullCryptoModule (no crypto) or the DefaultCryptoModule.
+
+ // If it's null, we won't have any parameters whatsoever. First, let's attempt to read
+ // parameters
+ Map<String,String> opts = new HashMap<String,String>();
+ int count = input.readInt();
+ for (int i = 0; i < count; i++) {
+ String key = input.readUTF();
+ String value = input.readUTF();
+ opts.put(key, value);
+ }
+
+ if (opts.size() == 0) {
+ // NullCryptoModule, we're done
+ decryptingInput = input;
+ } else {
+
+ // The DefaultCryptoModule will want to read the parameters from the underlying file, so we will put the file back to that spot.
+ org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory
+ .getCryptoModule(DefaultCryptoModule.class.getName());
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ input.seek(0);
+ input.readFully(magicBufferV2);
+ params.setEncryptedInputStream(input);
+
+ params = cryptoModule.getDecryptingInputStream(params);
+ if (params.getPlaintextInputStream() instanceof DataInputStream) {
+ decryptingInput = (DataInputStream) params.getPlaintextInputStream();
+ } else {
+ decryptingInput = new DataInputStream(params.getPlaintextInputStream());
+ }
+ }
+
+ } else {
+
+ input.seek(0);
+ decryptingInput = input;
+ }
+
+ }
+ } catch (EOFException e) {
+ log.warn("Got EOFException trying to read WAL header information, assuming the rest of the file (" + path + ") has no data.");
+ // A TabletServer might have died before the (complete) header was written
+ throw new LogHeaderIncompleteException(e);
+ }
+
+ return new DFSLoggerInputStreams(input, decryptingInput);
+ }
+
+ public synchronized void open(String address) throws IOException {
+ String filename = UUID.randomUUID().toString();
+ String logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
+
+ log.debug("DfsLogger.open() begin");
+ VolumeManager fs = conf.getFileSystem();
+
+ logPath = fs.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
+
+ metaReference = toString();
+ try {
+ short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
+ if (replication == 0)
+ replication = fs.getDefaultReplication(new Path(logPath));
+ long blockSize = conf.getConfiguration().getMemoryInBytes(Property.TSERV_WAL_BLOCKSIZE);
+ if (blockSize == 0)
+ blockSize = (long) (conf.getConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1);
+ if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
+ logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
+ else
+ logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
+
+ String syncMethod = conf.getConfiguration().get(Property.TSERV_WAL_SYNC_METHOD);
+ try {
+ // hsync: send data to datanodes and sync the data to disk
+ sync = logFile.getClass().getMethod(syncMethod);
+ } catch (Exception ex) {
+ log.warn("Could not find configured " + syncMethod + " method, trying to fall back to old Hadoop sync method", ex);
+
+ try {
+ // sync: send data to datanodes
+ sync = logFile.getClass().getMethod("sync");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // Initialize the crypto operations.
+ org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf
+ .getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
+
+ // Initialize the log file with a header and the crypto params used to set up this log file.
- logFile.write(LOG_FILE_HEADER_V3.getBytes(Constants.UTF8));
++ logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
+
+ NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
+ params.setPlaintextOutputStream(nfos);
+
+ // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
+ // so that that crypto module can re-read its own parameters.
+
+ logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
+
+ params = cryptoModule.getEncryptingOutputStream(params);
+ OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
+
+ // If the module just kicks back our original stream, then just use it, don't wrap it in
+ // another data OutputStream.
+ if (encipheringOutputStream == nfos) {
+ log.debug("No enciphering, using raw output stream");
+ encryptingLogFile = nfos;
+ } else {
+ log.debug("Enciphering found, wrapping in DataOutputStream");
+ encryptingLogFile = new DataOutputStream(encipheringOutputStream);
+ }
+
+ LogFileKey key = new LogFileKey();
+ key.event = OPEN;
+ key.tserverSession = filename;
+ key.filename = filename;
+ write(key, EMPTY);
+ sync.invoke(logFile);
+ log.debug("Got new write-ahead log: " + this);
+ } catch (Exception ex) {
+ if (logFile != null)
+ logFile.close();
+ logFile = null;
+ encryptingLogFile = null;
+ throw new IOException(ex);
+ }
+
+ syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
+ syncThread.setName("Accumulo WALog thread " + toString());
+ syncThread.start();
+ }
+
+ @Override
+ public String toString() {
+ String fileName = getFileName();
+ if (fileName.contains(":"))
+ return getLogger() + "/" + getFileName();
+ return fileName;
+ }
+
+ /**
+ * get the cq needed to reference this logger's entry in +r/!0
+ */
+ public String getMeta() {
+ if (null == metaReference) {
+ throw new IllegalStateException("logger doesn't have meta reference. " + this);
+ }
+ return metaReference;
+ }
+
+ public String getFileName() {
+ return logPath.toString();
+ }
+
+ public void close() throws IOException {
+
+ synchronized (closeLock) {
+ if (closed)
+ return;
+ // after closed is set to true, nothing else should be added to the queue
+ // CLOSED_MARKER should be the last thing on the queue, therefore when the
+ // background thread sees the marker and exits there should be nothing else
+ // to process... so nothing should be left waiting for the background
+ // thread to do work
+ closed = true;
+ workQueue.add(CLOSED_MARKER);
+ }
+
+ // wait for background thread to finish before closing log file
+ if (syncThread != null) {
+ try {
+ syncThread.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // expect workq should be empty at this point
+ if (workQueue.size() != 0) {
+ log.error("WAL work queue not empty after sync thread exited");
+ throw new IllegalStateException("WAL work queue not empty after sync thread exited");
+ }
+
+ if (encryptingLogFile != null)
+ try {
+ logFile.close();
+ } catch (IOException ex) {
+ log.error(ex);
+ throw new LogClosedException();
+ }
+ }
+
+ public synchronized void defineTablet(int seq, int tid, KeyExtent tablet) throws IOException {
+ // write this log to the METADATA table
+ final LogFileKey key = new LogFileKey();
+ key.event = DEFINE_TABLET;
+ key.seq = seq;
+ key.tid = tid;
+ key.tablet = tablet;
+ try {
+ write(key, EMPTY);
+ sync.invoke(logFile);
+ } catch (IllegalArgumentException e) {
+ log.error("Signature of sync method changed. Accumulo is likely incompatible with this version of Hadoop.");
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ log.error("Could not invoke sync method due to permission error.");
+ throw new RuntimeException(e);
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException) {
+ throw (IOException) cause;
+ } else if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ } else if (cause instanceof Error) {
+ throw (Error) cause;
+ } else {
+ // Cause is null, or some other checked exception that was added later.
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private synchronized void write(LogFileKey key, LogFileValue value) throws IOException {
+ key.write(encryptingLogFile);
+ value.write(encryptingLogFile);
+ encryptingLogFile.flush();
+ }
+
+ public LoggerOperation log(int seq, int tid, Mutation mutation) throws IOException {
+ return logManyTablets(Collections.singletonList(new TabletMutations(tid, seq, Collections.singletonList(mutation))));
+ }
+
+ private LoggerOperation logFileData(List<Pair<LogFileKey,LogFileValue>> keys) throws IOException {
+ DfsLogger.LogWork work = new DfsLogger.LogWork(new CountDownLatch(1));
+ synchronized (DfsLogger.this) {
+ try {
+ for (Pair<LogFileKey,LogFileValue> pair : keys) {
+ write(pair.getFirst(), pair.getSecond());
+ }
+ } catch (ClosedChannelException ex) {
+ throw new LogClosedException();
+ } catch (Exception e) {
+ log.error(e, e);
+ work.exception = e;
+ }
+ }
+
+ synchronized (closeLock) {
+ // use a different lock for close check so that adding to work queue does not need
+ // to wait on walog I/O operations
+
+ if (closed)
+ throw new LogClosedException();
+ workQueue.add(work);
+ }
+
+ return new LoggerOperation(work);
+ }
+
+ public LoggerOperation logManyTablets(List<TabletMutations> mutations) throws IOException {
+ List<Pair<LogFileKey,LogFileValue>> data = new ArrayList<Pair<LogFileKey,LogFileValue>>();
+ for (TabletMutations tabletMutations : mutations) {
+ LogFileKey key = new LogFileKey();
+ key.event = MANY_MUTATIONS;
+ key.seq = tabletMutations.getSeq();
+ key.tid = tabletMutations.getTid();
+ LogFileValue value = new LogFileValue();
+ value.mutations = tabletMutations.getMutations();
+ data.add(new Pair<LogFileKey,LogFileValue>(key, value));
+ }
+ return logFileData(data);
+ }
+
+ public LoggerOperation minorCompactionFinished(int seq, int tid, String fqfn) throws IOException {
+ LogFileKey key = new LogFileKey();
+ key.event = COMPACTION_FINISH;
+ key.seq = seq;
+ key.tid = tid;
+ return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
+ }
+
+ public LoggerOperation minorCompactionStarted(int seq, int tid, String fqfn) throws IOException {
+ LogFileKey key = new LogFileKey();
+ key.event = COMPACTION_START;
+ key.seq = seq;
+ key.tid = tid;
+ key.filename = fqfn;
+ return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
+ }
+
+ public String getLogger() {
+ String parts[] = logPath.split("/");
+ return StringUtil.join(Arrays.asList(parts[parts.length - 2].split("[+]")), ":");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
index 997f71b,0000000..2c807fd
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
@@@ -1,96 -1,0 +1,97 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.logger;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.hadoop.io.Writable;
+
+public class LogFileValue implements Writable {
+
+ private static final List<Mutation> empty = Collections.emptyList();
+
+ public List<Mutation> mutations = empty;
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int count = in.readInt();
+ mutations = new ArrayList<Mutation>(count);
+ for (int i = 0; i < count; i++) {
+ ServerMutation mutation = new ServerMutation();
+ mutation.readFields(in);
+ mutations.add(mutation);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(mutations.size());
+ for (Mutation m : mutations) {
+ m.write(out);
+ }
+ }
+
+ public static void print(LogFileValue value) {
+ System.out.println(value.toString());
+ }
+
+ private static String displayLabels(byte[] labels) {
- String s = new String(labels, Constants.UTF8);
++ String s = new String(labels, UTF_8);
+ s = s.replace("&", " & ");
+ s = s.replace("|", " | ");
+ return s;
+ }
+
+ public static String format(LogFileValue lfv, int maxMutations) {
+ if (lfv.mutations.size() == 0)
+ return "";
+ StringBuilder builder = new StringBuilder();
+ builder.append(lfv.mutations.size() + " mutations:\n");
+ int i = 0;
+ for (Mutation m : lfv.mutations) {
+ if (i++ >= maxMutations) {
+ builder.append("...");
+ break;
+ }
- builder.append(" ").append(new String(m.getRow(), Constants.UTF8)).append("\n");
++ builder.append(" ").append(new String(m.getRow(), UTF_8)).append("\n");
+ for (ColumnUpdate update : m.getUpdates()) {
+ String value = new String(update.getValue());
- builder.append(" ").append(new String(update.getColumnFamily(), Constants.UTF8)).append(":")
- .append(new String(update.getColumnQualifier(), Constants.UTF8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
++ builder.append(" ").append(new String(update.getColumnFamily(), UTF_8)).append(":")
++ .append(new String(update.getColumnQualifier(), UTF_8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
+ .append(update.getTimestamp()).append(" [").append(displayLabels(update.getColumnVisibility())).append("] ")
+ .append(update.isDeleted() ? "<deleted>" : value).append("\n");
+ }
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public String toString() {
+ return format(this, 5);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index c3f4fd0,0000000..29eefc8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -1,178 -1,0 +1,179 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.logger;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
+import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException;
+import org.apache.accumulo.tserver.log.MultiReader;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+ private static final Logger log = Logger.getLogger(LogReader.class);
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ VolumeManager fs = VolumeManagerImpl.get();
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ DFSLoggerInputStreams streams;
+ try {
+ streams = DfsLogger.readHeaderAndReturnStream(fs, path, ServerConfiguration.getSiteConfiguration());
+ } catch (LogHeaderIncompleteException e) {
+ log.warn("Could not read header for " + path + ". Ignoring...");
+ continue;
+ }
+ DataInputStream input = streams.getDecryptingInputStream();
+
+ try {
+ while (true) {
+ try {
+ key.readFields(input);
+ value.readFields(input);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ input.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, path);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
- rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
++ rowMatcher.reset(new String(m.getRow(), UTF_8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
index a679c48,ef6311b..3c20858
--- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
@@@ -71,98 -72,71 +72,98 @@@ public class TestBinaryRows
return l;
}
- static class Opts extends ClientOnRequiredTable {
+ public static class Opts extends ClientOnRequiredTable {
@Parameter(names="--mode", description="either 'ingest', 'delete', 'randomLookups', 'split', 'verify', 'verifyDeleted'", required=true)
- String mode;
+ public String mode;
@Parameter(names="--start", description="the lowest numbered row")
- long start = 0;
+ public long start = 0;
@Parameter(names="--count", description="number of rows to ingest", required=true)
- long num = 0;
+ public long num = 0;
}
- public static void main(String[] args) {
- Opts opts = new Opts();
- BatchWriterOpts bwOpts = new BatchWriterOpts();
- ScannerOpts scanOpts = new ScannerOpts();
- opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
+ public static void runTest(Connector connector, Opts opts, BatchWriterOpts bwOpts, ScannerOpts scanOpts) throws Exception {
- try {
- Connector connector = opts.getConnector();
-
- final Text CF = new Text("cf"), CQ = new Text("cq");
- final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
+ final Text CF = new Text("cf"), CQ = new Text("cq");
- final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
++ final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
+ if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+ boolean delete = opts.mode.equals("delete");
- if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
- BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
- boolean delete = opts.mode.equals("delete");
-
- for (long i = 0; i < opts.num; i++) {
- byte[] row = encodeLong(i + opts.start);
- String value = "" + (i + opts.start);
-
- Mutation m = new Mutation(new Text(row));
- if (delete) {
- m.putDelete(CF, CQ);
- } else {
- m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
- }
- bw.addMutation(m);
+ for (long i = 0; i < opts.num; i++) {
+ byte[] row = encodeLong(i + opts.start);
+ String value = "" + (i + opts.start);
+
+ Mutation m = new Mutation(new Text(row));
+ if (delete) {
+ m.putDelete(CF, CQ);
+ } else {
- m.put(CF, CQ, new Value(value.getBytes(Constants.UTF8)));
++ m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
}
-
- bw.close();
- } else if (opts.mode.equals("verifyDeleted")) {
- Scanner s = connector.createScanner(opts.tableName, opts.auths);
- s.setBatchSize(scanOpts.scanBatchSize);
- Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
- Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
- s.setBatchSize(50000);
- s.setRange(new Range(startKey, stopKey));
-
- for (Entry<Key,Value> entry : s) {
- System.err.println("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
- System.err.println("exiting...");
- System.exit(1);
- }
-
- } else if (opts.mode.equals("verify")) {
- long t1 = System.currentTimeMillis();
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ } else if (opts.mode.equals("verifyDeleted")) {
+ Scanner s = connector.createScanner(opts.tableName, opts.auths);
+ s.setBatchSize(scanOpts.scanBatchSize);
+ Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
+ Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
+ s.setBatchSize(50000);
+ s.setRange(new Range(startKey, stopKey));
+
+ for (Entry<Key,Value> entry : s) {
+ throw new Exception("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
+ }
+
+ } else if (opts.mode.equals("verify")) {
+ long t1 = System.currentTimeMillis();
+
+ Scanner s = connector.createScanner(opts.tableName, opts.auths);
+ Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
+ Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
+ s.setBatchSize(scanOpts.scanBatchSize);
+ s.setRange(new Range(startKey, stopKey));
+
+ long i = opts.start;
+
+ for (Entry<Key,Value> e : s) {
+ Key k = e.getKey();
+ Value v = e.getValue();
+
+ checkKeyValue(i, k, v);
+
+ i++;
+ }
+
+ if (i != opts.start + opts.num) {
+ throw new Exception("ERROR : did not see expected number of rows, saw " + (i - opts.start) + " expected " + opts.num);
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ System.out.printf("time : %9.2f secs%n", ((t2 - t1) / 1000.0));
+ System.out.printf("rate : %9.2f entries/sec%n", opts.num / ((t2 - t1) / 1000.0));
+
+ } else if (opts.mode.equals("randomLookups")) {
+ int numLookups = 1000;
+
+ Random r = new Random();
+
+ long t1 = System.currentTimeMillis();
+
+ for (int i = 0; i < numLookups; i++) {
+ long row = ((r.nextLong() & 0x7fffffffffffffffl) % opts.num) + opts.start;
Scanner s = connector.createScanner(opts.tableName, opts.auths);
- Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
- Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
s.setBatchSize(scanOpts.scanBatchSize);
+ Key startKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
+ Key stopKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], 0);
s.setRange(new Range(startKey, stopKey));
- long i = opts.start;
+ Iterator<Entry<Key,Value>> si = s.iterator();
- for (Entry<Key,Value> e : s) {
+ if (si.hasNext()) {
+ Entry<Key,Value> e = si.next();
Key k = e.getKey();
Value v = e.getValue();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestIngest.java
index bc3ff05,5d8e268..c3e77af
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@@ -16,16 -16,14 +16,17 @@@
*/
package org.apache.accumulo.test;
+ import static com.google.common.base.Charsets.UTF_8;
+
+import java.io.IOException;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index 7a848a3,125ef5f..16f0b3f
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@@ -16,11 -16,11 +16,12 @@@
*/
package org.apache.accumulo.test;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.util.ArrayList;
+import java.util.List;
import java.util.Map.Entry;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
@@@ -101,8 -97,8 +102,8 @@@ public class TestMultiTableIngest
// populate
for (int i = 0; i < opts.count; i++) {
- Mutation m = new Mutation(new Text(String.format("%05d", i)));
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
+ m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
b.getBatchWriter(tableNames.get(i % tableNames.size())).addMutation(m);
}
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
index 6082ebe,5962220..4f88c1b
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
@@@ -26,7 -28,7 +28,6 @@@ import java.util.Map.Entry
import java.util.Random;
import java.util.zip.CRC32;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
index 4bd07ca,6124a3d..11d19c5
--- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
@@@ -82,9 -82,9 +83,9 @@@ public class UndefinedAnalyzer
parseLog(log);
}
}
-
+
private void parseLog(File log) throws Exception {
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), Constants.UTF8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), UTF_8));
String line;
TreeMap<Long,Long> tm = null;
try {
@@@ -173,10 -172,10 +174,10 @@@
SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM");
String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + "";
String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + "";
-
+
for (File masterLog : masterLogs) {
-
+
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), Constants.UTF8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), UTF_8));
String line;
try {
while ((line = reader.readLine()) != null) {
@@@ -256,11 -252,11 +257,11 @@@
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
BatchScannerOpts bsOpts = new BatchScannerOpts();
- opts.parseArgs(UndefinedAnalyzer.class.getName(), args, opts);
-
+ opts.parseArgs(UndefinedAnalyzer.class.getName(), args, bsOpts);
+
List<UndefinedNode> undefs = new ArrayList<UndefinedNode>();
-
+
- BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, Constants.UTF8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, UTF_8));
String line;
while ((line = reader.readLine()) != null) {
String[] tokens = line.split("\\s");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index f26c8d7,166e2ad..949fc52
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -16,6 -16,10 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index cb17340,33ef0b5..72efee3
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -24,7 -26,7 +26,6 @@@ import java.util.Random
import java.util.TreeSet;
import java.util.UUID;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
@@@ -104,10 -100,10 +105,10 @@@ public class MetadataBatchScanTest
String dir = "/t-" + UUID.randomUUID();
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
- Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
for (int i = 0; i < 5; i++) {
- mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
- mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
++ mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
}
bw.addMutation(mut);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
index 5756934,0191b50..e4ce3f3
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
@@@ -41,9 -37,7 +43,8 @@@ import javax.xml.parsers.DocumentBuilde
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.log4j.Level;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@@ -201,163 -193,109 +202,163 @@@ public class Module extends Node
if (fixture != null) {
fixture.setUp(state);
}
-
- Node initNode = getNode(initNodeId);
-
- boolean test = false;
- if (initNode instanceof Test) {
- startTimer(initNode);
- test = true;
- }
- initNode.visit(state, getProps(initNodeId));
- if (test)
- stopTimer(initNode);
-
- state.visitedNode();
- // update aliases
- Set<String> aliases;
- if ((aliases = aliasMap.get(initNodeId)) != null)
- for (String alias : aliases) {
- ((Alias) nodes.get(alias)).update(initNodeId);
- }
-
- String curNodeId = initNodeId;
- int numHops = 0;
- long startTime = System.currentTimeMillis() / 1000;
- while (true) {
- // check if END state was reached
- if (curNodeId.equalsIgnoreCase("END")) {
- log.debug("reached END state");
- break;
- }
- // check if maxSec was reached
- long curTime = System.currentTimeMillis() / 1000;
- if ((curTime - startTime) > maxSec) {
- log.debug("reached maxSec(" + maxSec + ")");
- break;
- }
- // check if maxHops was reached
- if (numHops > maxHops) {
- log.debug("reached maxHops(" + maxHops + ")");
- break;
- }
- numHops++;
-
- if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
- throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
- }
- AdjList adj = adjMap.get(curNodeId);
- String nextNodeId = adj.randomNeighbor();
- Node nextNode = getNode(nextNodeId);
- if (nextNode instanceof Alias) {
- nextNodeId = ((Alias) nextNode).getTargetId();
- nextNode = ((Alias) nextNode).get();
+
+ ExecutorService service = new SimpleThreadPool(1, "RandomWalk Runner");
+
+ try {
+ Node initNode = getNode(initNodeId);
+
+ boolean test = false;
+ if (initNode instanceof Test) {
+ startTimer(initNode);
+ test = true;
}
- Properties nodeProps = getProps(nextNodeId);
- try {
- test = false;
- if (nextNode instanceof Test) {
- startTimer(nextNode);
- test = true;
+ initNode.visit(state, getProps(initNodeId));
+ if (test)
+ stopTimer(initNode);
+
+ state.visitedNode();
+ // update aliases
+ Set<String> aliases;
+ if ((aliases = aliasMap.get(initNodeId)) != null)
+ for (String alias : aliases) {
+ ((Alias) nodes.get(alias)).update(initNodeId);
}
- nextNode.visit(state, nodeProps);
- if (test)
- stopTimer(nextNode);
- } catch (Exception e) {
- log.debug("Connector belongs to user: " + state.getConnector().whoami());
- log.debug("Exception occured at: " + System.currentTimeMillis());
- log.debug("Properties for node: " + nextNodeId);
- for (Entry<Object,Object> entry : nodeProps.entrySet()) {
- log.debug(" " + entry.getKey() + ": " + entry.getValue());
+
+ String curNodeId = initNodeId;
+ int numHops = 0;
+ long startTime = System.currentTimeMillis() / 1000;
+ while (true) {
+ // check if END state was reached
+ if (curNodeId.equalsIgnoreCase("END")) {
+ log.debug("reached END state");
+ break;
}
- log.debug("Overall Properties");
- for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
- log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ // check if maxSec was reached
+ long curTime = System.currentTimeMillis() / 1000;
+ if ((curTime - startTime) > maxSec) {
+ log.debug("reached maxSec(" + maxSec + ")");
+ break;
}
- log.debug("State information");
- for (String key : new TreeSet<String>(state.getMap().keySet())) {
- Object value = state.getMap().get(key);
- String logMsg = " " + key + ": ";
- if (value == null)
- logMsg += "null";
- else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
- logMsg += value;
- else if (value instanceof byte[])
- logMsg += new String((byte[])value, UTF_8);
- else if (value instanceof PasswordToken)
- logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
- else
- logMsg += value.getClass()+ " - " + value;
-
- log.debug(logMsg);
+
+ // The number of seconds before the test should exit
+ long secondsRemaining = maxSec - (curTime - startTime);
+
+ // check if maxHops was reached
+ if (numHops > maxHops) {
+ log.debug("reached maxHops(" + maxHops + ")");
+ break;
}
- throw new Exception("Error running node " + nextNodeId, e);
- }
- state.visitedNode();
-
- // update aliases
- if ((aliases = aliasMap.get(curNodeId)) != null)
- for (String alias : aliases) {
- ((Alias) nodes.get(alias)).update(curNodeId);
+ numHops++;
+
+ if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
+ throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
+ }
+ AdjList adj = adjMap.get(curNodeId);
+ String nextNodeId = adj.randomNeighbor();
+ final Node nextNode;
+ Node nextNodeOrAlias = getNode(nextNodeId);
+ if (nextNodeOrAlias instanceof Alias) {
+ nextNodeId = ((Alias) nextNodeOrAlias).getTargetId();
+ nextNode = ((Alias) nextNodeOrAlias).get();
+ } else {
+ nextNode = nextNodeOrAlias;
+ }
+ final Properties nodeProps = getProps(nextNodeId);
+ try {
+ test = false;
+ if (nextNode instanceof Test) {
+ startTimer(nextNode);
+ test = true;
+ }
+
+ // Wrap the visit of the next node in the module in a callable that returns a thrown exception
+ FutureTask<Exception> task = new FutureTask<Exception>(new Callable<Exception>() {
+
+ @Override
+ public Exception call() throws Exception {
+ try {
+ nextNode.visit(state, nodeProps);
+ return null;
+ } catch (Exception e) {
+ return e;
+ }
+ }
+
+ });
+
+ // Run the task (should execute immediately)
+ service.submit(task);
+
+ Exception nodeException;
+ try {
+ // Bound the time we'll wait for the node to complete
+ nodeException = task.get(secondsRemaining, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ log.warn("Interrupted waiting for " + nextNode.getClass().getSimpleName() + " to complete. Exiting.", e);
+ break;
+ } catch (ExecutionException e) {
+ log.error("Caught error executing " + nextNode.getClass().getSimpleName(), e);
+ throw e;
+ } catch (TimeoutException e) {
+ log.info("Timed out waiting for " + nextNode.getClass().getSimpleName() + " to complete (waited " + secondsRemaining + " seconds). Exiting.", e);
+ break;
+ }
+
+ // The RandomWalk node throw an Exception that that Callable handed back
+ // Throw it and let the Module perform cleanup
+ if (null != nodeException) {
+ throw nodeException;
+ }
+
+ if (test)
+ stopTimer(nextNode);
+ } catch (Exception e) {
+ log.debug("Connector belongs to user: " + state.getConnector().whoami());
+ log.debug("Exception occured at: " + System.currentTimeMillis());
+ log.debug("Properties for node: " + nextNodeId);
+ for (Entry<Object,Object> entry : nodeProps.entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
+ log.debug("Overall Properties");
+ for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
+ log.debug("State information");
+ for (String key : new TreeSet<String>(state.getMap().keySet())) {
+ Object value = state.getMap().get(key);
+ String logMsg = " " + key + ": ";
+ if (value == null)
+ logMsg += "null";
+ else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
+ logMsg += value;
+ else if (value instanceof byte[])
- logMsg += new String((byte[]) value, Constants.UTF8);
++ logMsg += new String((byte[]) value, UTF_8);
+ else if (value instanceof PasswordToken)
- logMsg += new String(((PasswordToken) value).getPassword(), Constants.UTF8);
++ logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
+ else
+ logMsg += value.getClass() + " - " + value;
+
+ log.debug(logMsg);
+ }
+ throw new Exception("Error running node " + nextNodeId, e);
}
-
- curNodeId = nextNodeId;
+ state.visitedNode();
+
+ // update aliases
+ if ((aliases = aliasMap.get(curNodeId)) != null)
+ for (String alias : aliases) {
+ ((Alias) nodes.get(alias)).update(curNodeId);
+ }
+
+ curNodeId = nextNodeId;
+ }
+ } finally {
+ if (null != service) {
+ service.shutdownNow();
+ }
}
-
+
if (teardown && (fixture != null)) {
log.debug("tearing down module");
fixture.tearDown(state);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
index 1704e49,05cc4f0..67438d9
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
@@@ -20,10 -21,10 +21,10 @@@ import static com.google.common.base.Ch
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.test.randomwalk.State;
-public class BulkMinusOne extends BulkTest {
-
+public class BulkMinusOne extends BulkImportTest {
+
- private static final Value negOne = new Value("-1".getBytes(Constants.UTF8));
+ private static final Value negOne = new Value("-1".getBytes(UTF_8));
-
+
@Override
protected void runLater(State state) throws Exception {
log.info("Decrementing");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
index f002274,7f9a218..3dcff6e
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
@@@ -51,10 -52,10 +52,10 @@@ public class BatchWrite extends Test
try {
int numRows = rand.nextInt(100000);
for (int i = 0; i < numRows; i++) {
- Mutation m = new Mutation(String.format("%016x", (rand.nextLong() & 0x7fffffffffffffffl)));
- long val = (rand.nextLong() & 0x7fffffffffffffffl);
+ Mutation m = new Mutation(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl));
+ long val = rand.nextLong() & 0x7fffffffffffffffl;
for (int j = 0; j < 10; j++) {
- m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(Constants.UTF8)));
+ m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(UTF_8)));
}
bw.addMutation(m);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
index c8da3d8,c227a11..b3519b6
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
@@@ -47,8 -50,8 +49,8 @@@ public class StopTabletServer extends T
Collections.sort(children);
Stat stat = new Stat();
byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
- if (!"master".equals(new String(data, Constants.UTF8))) {
+ if (!"master".equals(new String(data, UTF_8))) {
- result.add(new TServerInstance(AddressUtil.parseAddress(child, Property.TSERV_CLIENTPORT), stat.getEphemeralOwner()));
+ result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
}
}
} catch (KeeperException.NoNodeException ex) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
[31/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index 9537f4e,0000000..83e83cf
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -1,1315 -1,0 +1,1317 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.ThriftTransportPool;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Iface;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Processor;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.master.thrift.MasterState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.AgeOffStore;
+import org.apache.accumulo.fate.Fate;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.master.recovery.RecoveryManager;
+import org.apache.accumulo.master.state.TableCounts;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
+import org.apache.accumulo.server.master.balancer.TabletBalancer;
+import org.apache.accumulo.server.master.state.CurrentState;
+import org.apache.accumulo.server.master.state.DeadServerList;
+import org.apache.accumulo.server.master.state.MergeInfo;
+import org.apache.accumulo.server.master.state.MergeState;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.RootTabletStateStore;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.master.state.TabletState;
+import org.apache.accumulo.server.master.state.ZooStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.security.handler.ZKPermHandler;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.tables.TableObserver;
+import org.apache.accumulo.server.util.DefaultMap;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.RpcWrapper;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * The Master is responsible for assigning and balancing tablets to tablet servers.
+ *
+ * The master will also coordinate log recoveries and reports general status.
+ */
+public class Master implements LiveTServerSet.Listener, TableObserver, CurrentState {
+
+ final static Logger log = Logger.getLogger(Master.class);
+
+ final static int ONE_SECOND = 1000;
+ final private static Text METADATA_TABLE_ID = new Text(MetadataTable.ID);
+ final private static Text ROOT_TABLE_ID = new Text(RootTable.ID);
+ final static long TIME_TO_WAIT_BETWEEN_SCANS = 60 * ONE_SECOND;
+ final private static long TIME_BETWEEN_MIGRATION_CLEANUPS = 5 * 60 * ONE_SECOND;
+ final static long WAIT_BETWEEN_ERRORS = ONE_SECOND;
+ final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;
+ final private static int MAX_CLEANUP_WAIT_TIME = ONE_SECOND;
+ final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = ONE_SECOND;
+ final static int MAX_TSERVER_WORK_CHUNK = 5000;
+ final private static int MAX_BAD_STATUS_COUNT = 3;
+
+ final VolumeManager fs;
+ final private Instance instance;
+ final private String hostname;
+ final LiveTServerSet tserverSet;
+ final private List<TabletGroupWatcher> watchers = new ArrayList<TabletGroupWatcher>();
+ final SecurityOperation security;
+ final Map<TServerInstance,AtomicInteger> badServers = Collections.synchronizedMap(new DefaultMap<TServerInstance,AtomicInteger>(new AtomicInteger()));
+ final Set<TServerInstance> serversToShutdown = Collections.synchronizedSet(new HashSet<TServerInstance>());
+ final SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,TServerInstance>());
+ final EventCoordinator nextEvent = new EventCoordinator();
+ final private Object mergeLock = new Object();
+ RecoveryManager recoveryManager = null;
+
+ ZooLock masterLock = null;
+ private TServer clientService = null;
+ TabletBalancer tabletBalancer;
+
+ private MasterState state = MasterState.INITIAL;
+
+ Fate<Master> fate;
+
+ volatile SortedMap<TServerInstance,TabletServerStatus> tserverStatus = Collections.unmodifiableSortedMap(new TreeMap<TServerInstance,TabletServerStatus>());
+
+ synchronized MasterState getMasterState() {
+ return state;
+ }
+
+ public boolean stillMaster() {
+ return getMasterState() != MasterState.STOP;
+ }
+
+ static final boolean X = true;
+ static final boolean _ = false;
+ // @formatter:off
+ static final boolean transitionOK[][] = {
+ // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
+ /* INITIAL */ {X, X, _, _, _, _, X},
+ /* HAVE_LOCK */ {_, X, X, X, _, _, X},
+ /* SAFE_MODE */ {_, _, X, X, X, _, X},
+ /* NORMAL */ {_, _, X, X, X, _, X},
+ /* UNLOAD_METADATA_TABLETS */ {_, _, X, X, X, X, X},
+ /* UNLOAD_ROOT_TABLET */ {_, _, _, X, X, X, X},
+ /* STOP */ {_, _, _, _, _, X, X}};
+ //@formatter:on
+ synchronized void setMasterState(MasterState newState) {
+ if (state.equals(newState))
+ return;
+ if (!transitionOK[state.ordinal()][newState.ordinal()]) {
+ log.error("Programmer error: master should not transition from " + state + " to " + newState);
+ }
+ MasterState oldState = state;
+ state = newState;
+ nextEvent.event("State changed from %s to %s", oldState, newState);
+ if (newState == MasterState.STOP) {
+ // Give the server a little time before shutdown so the client
+ // thread requesting the stop can return
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ // This frees the main thread and will cause the master to exit
+ clientService.stop();
+ Master.this.nextEvent.event("stopped event loop");
+ }
+
+ }, 100l, 1000l);
+ }
+
+ if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {
+ upgradeZookeeper();
+ }
+
+ if (oldState != newState && (newState == MasterState.NORMAL)) {
+ upgradeMetadata();
+ }
+ }
+
+ private void moveRootTabletToRootTable(IZooReaderWriter zoo) throws Exception {
+ String dirZPath = ZooUtil.getRoot(instance) + RootTable.ZROOT_TABLET_PATH;
+
+ if (!zoo.exists(dirZPath)) {
+ Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
+ if (fs.exists(oldPath)) {
+ String newPath = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
+ fs.mkdirs(new Path(newPath));
+ if (!fs.rename(oldPath, new Path(newPath))) {
+ throw new IOException("Failed to move root tablet from " + oldPath + " to " + newPath);
+ }
+
+ log.info("Upgrade renamed " + oldPath + " to " + newPath);
+ }
+
+ Path location = null;
+
+ for (String basePath : ServerConstants.getTablesDirs()) {
+ Path path = new Path(basePath + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION);
+ if (fs.exists(path)) {
+ if (location != null) {
+ throw new IllegalStateException("Root table at multiple locations " + location + " " + path);
+ }
+
+ location = path;
+ }
+ }
+
+ if (location == null)
+ throw new IllegalStateException("Failed to find root tablet");
+
+ log.info("Upgrade setting root table location in zookeeper " + location);
+ zoo.putPersistentData(dirZPath, location.toString().getBytes(), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ private boolean haveUpgradedZooKeeper = false;
+
+ private void upgradeZookeeper() {
+ // 1.5.1 and 1.6.0 both do some state checking after obtaining the zoolock for the
+ // monitor and before starting up. It's not tied to the data version at all (and would
+ // introduce unnecessary complexity to try to make the master do it), but be aware
+ // that the master is not the only thing that may alter zookeeper before starting.
+
+ final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
+ if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
+ // This Master hasn't started Fate yet, so any outstanding transactions must be from before the upgrade.
+ // Change to Guava's Verify once we use Guava 17.
+ if (null != fate) {
+ throw new IllegalStateException(
+ "Access to Fate should not have been initialized prior to the Master transitioning to active. Please save all logs and file a bug.");
+ }
+ Accumulo.abortIfFateTransactions();
+ try {
+ log.info("Upgrading zookeeper");
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ final String zooRoot = ZooUtil.getRoot(instance);
+
+ if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
+ log.debug("Handling updates for version " + ServerConstants.TWO_DATA_VERSIONS_AGO);
+
+ log.debug("Cleaning out remnants of logger role.");
+ zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
+
+ final byte[] zero = new byte[] {'0'};
+ log.debug("Initializing recovery area.");
+ zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
+
+ for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
+ log.debug("Prepping table " + id + " for compaction cancellations.");
+ zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
+ }
+ }
+
+ // create initial namespaces
+ String namespaces = ZooUtil.getRoot(instance) + Constants.ZNAMESPACES;
+ zoo.putPersistentData(namespaces, new byte[0], NodeExistsPolicy.SKIP);
+ for (Pair<String,String> namespace : Iterables.concat(
+ Collections.singleton(new Pair<String,String>(Namespaces.ACCUMULO_NAMESPACE, Namespaces.ACCUMULO_NAMESPACE_ID)),
+ Collections.singleton(new Pair<String,String>(Namespaces.DEFAULT_NAMESPACE, Namespaces.DEFAULT_NAMESPACE_ID)))) {
+ String ns = namespace.getFirst();
+ String id = namespace.getSecond();
+ log.debug("Upgrade creating namespace \"" + ns + "\" (ID: " + id + ")");
+ if (!Namespaces.exists(instance, id))
+ TableManager.prepareNewNamespaceState(instance.getInstanceID(), id, ns, NodeExistsPolicy.SKIP);
+ }
+
+ // create root table
+ log.debug("Upgrade creating table " + RootTable.NAME + " (ID: " + RootTable.ID + ")");
+ TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE,
+ NodeExistsPolicy.SKIP);
+ Initialize.initMetadataConfig();
+ // ensure root user can flush root table
+ security.grantTablePermission(SystemCredentials.get().toThrift(instance), security.getRootUsername(), RootTable.ID, TablePermission.ALTER_TABLE,
+ Namespaces.ACCUMULO_NAMESPACE_ID);
+
+ // put existing tables in the correct namespaces
+ String tables = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ for (String tableId : zoo.getChildren(tables)) {
+ String targetNamespace = (MetadataTable.ID.equals(tableId) || RootTable.ID.equals(tableId)) ? Namespaces.ACCUMULO_NAMESPACE_ID
+ : Namespaces.DEFAULT_NAMESPACE_ID;
- log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), Constants.UTF8) + " (ID: "
++ log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), UTF_8) + " (ID: "
+ + tableId + ") into namespace with ID " + targetNamespace);
- zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++ zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(UTF_8), NodeExistsPolicy.SKIP);
+ }
+
+ // rename metadata table
+ log.debug("Upgrade renaming table " + MetadataTable.OLD_NAME + " (ID: " + MetadataTable.ID + ") to " + MetadataTable.NAME);
- zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(Constants.UTF8),
++ zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8),
+ NodeExistsPolicy.OVERWRITE);
+
+ moveRootTabletToRootTable(zoo);
+
+ // add system namespace permissions to existing users
+ ZKPermHandler perm = new ZKPermHandler();
+ perm.initialize(instance.getInstanceID(), true);
+ String users = ZooUtil.getRoot(instance) + "/users";
+ for (String user : zoo.getChildren(users)) {
+ zoo.putPersistentData(users + "/" + user + "/Namespaces", new byte[0], NodeExistsPolicy.SKIP);
+ perm.grantNamespacePermission(user, Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.READ);
+ }
+ perm.grantNamespacePermission("root", Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.ALTER_TABLE);
+ haveUpgradedZooKeeper = true;
+ } catch (Exception ex) {
+ log.fatal("Error performing upgrade", ex);
+ System.exit(1);
+ }
+ }
+ }
+
+ private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
+ private final CountDownLatch waitForMetadataUpgrade = new CountDownLatch(1);
+
+ private final ServerConfiguration serverConfig;
+
+ private void upgradeMetadata() {
+ // we make sure we're only doing the rest of this method once so that we can signal to other threads that an upgrade wasn't needed.
+ if (upgradeMetadataRunning.compareAndSet(false, true)) {
+ final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
+ if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
+ // sanity check that we passed the Fate verification prior to ZooKeeper upgrade, and that Fate still hasn't been started.
+ // Change both to use Guava's Verify once we use Guava 17.
+ if (!haveUpgradedZooKeeper) {
+ throw new IllegalStateException(
+ "We should only attempt to upgrade Accumulo's metadata table if we've already upgraded ZooKeeper. Please save all logs and file a bug.");
+ }
+ if (null != fate) {
+ throw new IllegalStateException(
+ "Access to Fate should not have been initialized prior to the Master finishing upgrades. Please save all logs and file a bug.");
+ }
+ Runnable upgradeTask = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ log.info("Starting to upgrade metadata table.");
+ if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
+ log.info("Updating Delete Markers in metadata table for version 1.4");
+ MetadataTableUtil.moveMetaDeleteMarkersFrom14(instance, SystemCredentials.get());
+ } else {
+ log.info("Updating Delete Markers in metadata table.");
+ MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
+ }
+ log.info("Updating persistent data version.");
+ Accumulo.updateAccumuloVersion(fs, accumuloPersistentVersion);
+ log.info("Upgrade complete");
+ waitForMetadataUpgrade.countDown();
+ } catch (Exception ex) {
+ log.fatal("Error performing upgrade", ex);
+ System.exit(1);
+ }
+
+ }
+ };
+
+ // need to run this in a separate thread because a lock is held that prevents metadata tablets from being assigned and this task writes to the
+ // metadata table
+ new Thread(upgradeTask).start();
+ } else {
+ waitForMetadataUpgrade.countDown();
+ }
+ }
+ }
+
+ private int assignedOrHosted(Text tableId) {
+ int result = 0;
+ for (TabletGroupWatcher watcher : watchers) {
+ TableCounts count = watcher.getStats(tableId);
+ result += count.hosted() + count.assigned();
+ }
+ return result;
+ }
+
+ private int totalAssignedOrHosted() {
+ int result = 0;
+ for (TabletGroupWatcher watcher : watchers) {
+ for (TableCounts counts : watcher.getStats().values()) {
+ result += counts.assigned() + counts.hosted();
+ }
+ }
+ return result;
+ }
+
+ private int nonMetaDataTabletsAssignedOrHosted() {
+ return totalAssignedOrHosted() - assignedOrHosted(new Text(MetadataTable.ID)) - assignedOrHosted(new Text(RootTable.ID));
+ }
+
+ private int notHosted() {
+ int result = 0;
+ for (TabletGroupWatcher watcher : watchers) {
+ for (TableCounts counts : watcher.getStats().values()) {
+ result += counts.assigned() + counts.assignedToDeadServers();
+ }
+ }
+ return result;
+ }
+
+ // The number of unassigned tablets that should be assigned: displayed on the monitor page
+ int displayUnassigned() {
+ int result = 0;
+ switch (getMasterState()) {
+ case NORMAL:
+ // Count offline tablets for online tables
+ for (TabletGroupWatcher watcher : watchers) {
+ TableManager manager = TableManager.getInstance();
+ for (Entry<Text,TableCounts> entry : watcher.getStats().entrySet()) {
+ Text tableId = entry.getKey();
+ TableCounts counts = entry.getValue();
+ TableState tableState = manager.getTableState(tableId.toString());
+ if (tableState != null && tableState.equals(TableState.ONLINE)) {
+ result += counts.unassigned() + counts.assignedToDeadServers() + counts.assigned();
+ }
+ }
+ }
+ break;
+ case SAFE_MODE:
+ // Count offline tablets for the metadata table
+ for (TabletGroupWatcher watcher : watchers) {
+ result += watcher.getStats(METADATA_TABLE_ID).unassigned();
+ }
+ break;
+ case UNLOAD_METADATA_TABLETS:
+ case UNLOAD_ROOT_TABLET:
+ for (TabletGroupWatcher watcher : watchers) {
+ result += watcher.getStats(METADATA_TABLE_ID).unassigned();
+ }
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+
+ public void mustBeOnline(final String tableId) throws ThriftTableOperationException {
+ Tables.clearCache(instance);
+ if (!Tables.getTableState(instance, tableId).equals(TableState.ONLINE))
+ throw new ThriftTableOperationException(tableId, null, TableOperation.MERGE, TableOperationExceptionType.OFFLINE, "table is not online");
+ }
+
+ public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
+ return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ }
+
+ private Master(ServerConfiguration config, VolumeManager fs, String hostname) throws IOException {
+ this.serverConfig = config;
+ this.instance = config.getInstance();
+ this.fs = fs;
+ this.hostname = hostname;
+
+ AccumuloConfiguration aconf = serverConfig.getConfiguration();
+
+ log.info("Version " + Constants.VERSION);
+ log.info("Instance " + instance.getInstanceID());
+ ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ security = AuditedSecurityOperation.getInstance();
+ tserverSet = new LiveTServerSet(instance, config.getConfiguration(), this);
+ this.tabletBalancer = aconf.instantiateClassProperty(Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
+ this.tabletBalancer.init(serverConfig);
+
+ try {
+ AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() {
+ @Override
+ public Iterator<Entry<String,String>> iterator() {
+ return getSystemConfiguration().iterator();
+ }
+ }));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ public TServerConnection getConnection(TServerInstance server) {
+ return tserverSet.getConnection(server);
+ }
+
+ public MergeInfo getMergeInfo(Text tableId) {
+ synchronized (mergeLock) {
+ try {
+ String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
+ if (!ZooReaderWriter.getInstance().exists(path))
+ return new MergeInfo();
+ byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat());
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(data, data.length);
+ MergeInfo info = new MergeInfo();
+ info.readFields(in);
+ return info;
+ } catch (KeeperException.NoNodeException ex) {
+ log.info("Error reading merge state, it probably just finished");
+ return new MergeInfo();
+ } catch (Exception ex) {
+ log.warn("Unexpected error reading merge state", ex);
+ return new MergeInfo();
+ }
+ }
+ }
+
+ public void setMergeState(MergeInfo info, MergeState state) throws IOException, KeeperException, InterruptedException {
+ synchronized (mergeLock) {
+ String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + info.getExtent().getTableId().toString() + "/merge";
+ info.setState(state);
+ if (state.equals(MergeState.NONE)) {
+ ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
+ } else {
+ DataOutputBuffer out = new DataOutputBuffer();
+ try {
+ info.write(out);
+ } catch (IOException ex) {
+ throw new RuntimeException("Unlikely", ex);
+ }
+ ZooReaderWriter.getInstance().putPersistentData(path, out.getData(),
+ state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE);
+ }
+ mergeLock.notifyAll();
+ }
+ nextEvent.event("Merge state of %s set to %s", info.getExtent(), state);
+ }
+
+ public void clearMergeState(Text tableId) throws IOException, KeeperException, InterruptedException {
+ synchronized (mergeLock) {
+ String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
+ ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
+ mergeLock.notifyAll();
+ }
+ nextEvent.event("Merge state of %s cleared", tableId);
+ }
+
+ void setMasterGoalState(MasterGoalState state) {
+ try {
+ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, state.name().getBytes(),
+ NodeExistsPolicy.OVERWRITE);
+ } catch (Exception ex) {
+ log.error("Unable to set master goal state in zookeeper");
+ }
+ }
+
+ MasterGoalState getMasterGoalState() {
+ while (true)
+ try {
+ byte[] data = ZooReaderWriter.getInstance().getData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, null);
+ return MasterGoalState.valueOf(new String(data));
+ } catch (Exception e) {
+ log.error("Problem getting real goal state: " + e);
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public boolean hasCycled(long time) {
+ for (TabletGroupWatcher watcher : watchers) {
+ if (watcher.stats.lastScanFinished() < time)
+ return false;
+ }
+
+ return true;
+ }
+
+ public void clearMigrations(String tableId) {
+ synchronized (migrations) {
+ Iterator<KeyExtent> iterator = migrations.keySet().iterator();
+ while (iterator.hasNext()) {
+ KeyExtent extent = iterator.next();
+ if (extent.getTableId().toString().equals(tableId)) {
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+ static enum TabletGoalState {
+ HOSTED, UNASSIGNED, DELETED
+ };
+
+ TabletGoalState getSystemGoalState(TabletLocationState tls) {
+ switch (getMasterState()) {
+ case NORMAL:
+ return TabletGoalState.HOSTED;
+ case HAVE_LOCK: // fall-through intended
+ case INITIAL: // fall-through intended
+ case SAFE_MODE:
+ if (tls.extent.isMeta())
+ return TabletGoalState.HOSTED;
+ return TabletGoalState.UNASSIGNED;
+ case UNLOAD_METADATA_TABLETS:
+ if (tls.extent.isRootTablet())
+ return TabletGoalState.HOSTED;
+ return TabletGoalState.UNASSIGNED;
+ case UNLOAD_ROOT_TABLET:
+ return TabletGoalState.UNASSIGNED;
+ case STOP:
+ return TabletGoalState.UNASSIGNED;
+ default:
+ throw new IllegalStateException("Unknown Master State");
+ }
+ }
+
+ TabletGoalState getTableGoalState(KeyExtent extent) {
+ TableState tableState = TableManager.getInstance().getTableState(extent.getTableId().toString());
+ if (tableState == null)
+ return TabletGoalState.DELETED;
+ switch (tableState) {
+ case DELETING:
+ return TabletGoalState.DELETED;
+ case OFFLINE:
+ case NEW:
+ return TabletGoalState.UNASSIGNED;
+ default:
+ return TabletGoalState.HOSTED;
+ }
+ }
+
+ TabletGoalState getGoalState(TabletLocationState tls, MergeInfo mergeInfo) {
+ KeyExtent extent = tls.extent;
+ // Shutting down?
+ TabletGoalState state = getSystemGoalState(tls);
+ if (state == TabletGoalState.HOSTED) {
+ if (tls.current != null && serversToShutdown.contains(tls.current)) {
+ return TabletGoalState.UNASSIGNED;
+ }
+ // Handle merge transitions
+ if (mergeInfo.getExtent() != null) {
+ log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent));
+ if (mergeInfo.overlaps(extent)) {
+ switch (mergeInfo.getState()) {
+ case NONE:
+ case COMPLETE:
+ break;
+ case STARTED:
+ case SPLITTING:
+ return TabletGoalState.HOSTED;
+ case WAITING_FOR_CHOPPED:
+ if (tls.getState(onlineTabletServers()).equals(TabletState.HOSTED)) {
+ if (tls.chopped)
+ return TabletGoalState.UNASSIGNED;
+ } else {
+ if (tls.chopped && tls.walogs.isEmpty())
+ return TabletGoalState.UNASSIGNED;
+ }
+
+ return TabletGoalState.HOSTED;
+ case WAITING_FOR_OFFLINE:
+ case MERGING:
+ return TabletGoalState.UNASSIGNED;
+ }
+ }
+ }
+
+ // taking table offline?
+ state = getTableGoalState(extent);
+ if (state == TabletGoalState.HOSTED) {
+ // Maybe this tablet needs to be migrated
+ TServerInstance dest = migrations.get(extent);
+ if (dest != null && tls.current != null && !dest.equals(tls.current)) {
+ return TabletGoalState.UNASSIGNED;
+ }
+ }
+ }
+ return state;
+ }
+
+ private class MigrationCleanupThread extends Daemon {
+
+ @Override
+ public void run() {
+ setName("Migration Cleanup Thread");
+ while (stillMaster()) {
+ if (!migrations.isEmpty()) {
+ try {
+ cleanupOfflineMigrations();
+ cleanupNonexistentMigrations(getConnector());
+ } catch (Exception ex) {
+ log.error("Error cleaning up migrations", ex);
+ }
+ }
+ UtilWaitThread.sleep(TIME_BETWEEN_MIGRATION_CLEANUPS);
+ }
+ }
+
+ /**
+ * If a migrating tablet splits, and the tablet dies before sending the master a message, the migration will refer to a non-existing tablet, so it can never
+ * complete. Periodically scan the metadata table and remove any migrating tablets that no longer exist.
+ */
+ private void cleanupNonexistentMigrations(final Connector connector) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ Set<KeyExtent> found = new HashSet<KeyExtent>();
+ for (Entry<Key,Value> entry : scanner) {
+ KeyExtent extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
+ if (migrations.containsKey(extent)) {
+ found.add(extent);
+ }
+ }
+ migrations.keySet().retainAll(found);
+ }
+
+ /**
+ * If migrating a tablet for a table that is offline, the migration can never succeed because no tablet server will load the tablet. check for offline
+ * tables and remove their migrations.
+ */
+ private void cleanupOfflineMigrations() {
+ TableManager manager = TableManager.getInstance();
+ for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
+ TableState state = manager.getTableState(tableId);
+ if (TableState.OFFLINE == state) {
+ clearMigrations(tableId);
+ }
+ }
+ }
+ }
+
+ private class StatusThread extends Daemon {
+
+ @Override
+ public void run() {
+ setName("Status Thread");
+ EventCoordinator.Listener eventListener = nextEvent.getListener();
+ while (stillMaster()) {
+ long wait = DEFAULT_WAIT_FOR_WATCHER;
+ try {
+ switch (getMasterGoalState()) {
+ case NORMAL:
+ setMasterState(MasterState.NORMAL);
+ break;
+ case SAFE_MODE:
+ if (getMasterState() == MasterState.NORMAL) {
+ setMasterState(MasterState.SAFE_MODE);
+ }
+ if (getMasterState() == MasterState.HAVE_LOCK) {
+ setMasterState(MasterState.SAFE_MODE);
+ }
+ break;
+ case CLEAN_STOP:
+ switch (getMasterState()) {
+ case NORMAL:
+ setMasterState(MasterState.SAFE_MODE);
+ break;
+ case SAFE_MODE: {
+ int count = nonMetaDataTabletsAssignedOrHosted();
+ log.debug(String.format("There are %d non-metadata tablets assigned or hosted", count));
+ if (count == 0)
+ setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
+ }
+ break;
+ case UNLOAD_METADATA_TABLETS: {
+ int count = assignedOrHosted(METADATA_TABLE_ID);
+ log.debug(String.format("There are %d metadata tablets assigned or hosted", count));
+ if (count == 0)
+ setMasterState(MasterState.UNLOAD_ROOT_TABLET);
+ }
+ break;
+ case UNLOAD_ROOT_TABLET: {
+ int count = assignedOrHosted(METADATA_TABLE_ID);
+ if (count > 0) {
+ log.debug(String.format("%d metadata tablets online", count));
+ setMasterState(MasterState.UNLOAD_ROOT_TABLET);
+ }
+ int root_count = assignedOrHosted(ROOT_TABLE_ID);
+ if (root_count > 0)
+ log.debug("The root tablet is still assigned or hosted");
+ if (count + root_count == 0) {
+ Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
+ log.debug("stopping " + currentServers.size() + " tablet servers");
+ for (TServerInstance server : currentServers) {
+ try {
+ serversToShutdown.add(server);
+ tserverSet.getConnection(server).fastHalt(masterLock);
+ } catch (TException e) {
+ // its probably down, and we don't care
+ } finally {
+ tserverSet.remove(server);
+ }
+ }
+ if (currentServers.size() == 0)
+ setMasterState(MasterState.STOP);
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ wait = updateStatus();
+ eventListener.waitForEvents(wait);
+ } catch (Throwable t) {
+ log.error("Error balancing tablets", t);
+ UtilWaitThread.sleep(WAIT_BETWEEN_ERRORS);
+ }
+ }
+ }
+
+ private long updateStatus() throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ tserverStatus = Collections.synchronizedSortedMap(gatherTableInformation());
+ checkForHeldServer(tserverStatus);
+
+ if (!badServers.isEmpty()) {
+ log.debug("not balancing because the balance information is out-of-date " + badServers.keySet());
+ } else if (notHosted() > 0) {
+ log.debug("not balancing because there are unhosted tablets: " + notHosted());
+ } else if (getMasterGoalState() == MasterGoalState.CLEAN_STOP) {
+ log.debug("not balancing because the master is attempting to stop cleanly");
+ } else if (!serversToShutdown.isEmpty()) {
+ log.debug("not balancing while shutting down servers " + serversToShutdown);
+ } else {
+ return balanceTablets();
+ }
+ return DEFAULT_WAIT_FOR_WATCHER;
+ }
+
+ private void checkForHeldServer(SortedMap<TServerInstance,TabletServerStatus> tserverStatus) {
+ TServerInstance instance = null;
+ int crazyHoldTime = 0;
+ int someHoldTime = 0;
+ final long maxWait = getSystemConfiguration().getTimeInMillis(Property.TSERV_HOLD_TIME_SUICIDE);
+ for (Entry<TServerInstance,TabletServerStatus> entry : tserverStatus.entrySet()) {
+ if (entry.getValue().getHoldTime() > 0) {
+ someHoldTime++;
+ if (entry.getValue().getHoldTime() > maxWait) {
+ instance = entry.getKey();
+ crazyHoldTime++;
+ }
+ }
+ }
+ if (crazyHoldTime == 1 && someHoldTime == 1 && tserverStatus.size() > 1) {
+ log.warn("Tablet server " + instance + " exceeded maximum hold time: attempting to kill it");
+ try {
+ TServerConnection connection = tserverSet.getConnection(instance);
+ if (connection != null)
+ connection.fastHalt(masterLock);
+ } catch (TException e) {
+ log.error(e, e);
+ }
+ tserverSet.remove(instance);
+ }
+ }
+
+ private long balanceTablets() {
+ List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
+ Set<KeyExtent> migrationsCopy = new HashSet<KeyExtent>();
+ synchronized (migrations) {
+ migrationsCopy.addAll(migrations.keySet());
+ }
+ long wait = tabletBalancer.balance(Collections.unmodifiableSortedMap(tserverStatus), Collections.unmodifiableSet(migrationsCopy), migrationsOut);
+
+ for (TabletMigration m : TabletBalancer.checkMigrationSanity(tserverStatus.keySet(), migrationsOut)) {
+ if (migrations.containsKey(m.tablet)) {
+ log.warn("balancer requested migration more than once, skipping " + m);
+ continue;
+ }
+ migrations.put(m.tablet, m.newServer);
+ log.debug("migration " + m);
+ }
+ if (migrationsOut.size() > 0) {
+ nextEvent.event("Migrating %d more tablets, %d total", migrationsOut.size(), migrations.size());
+ }
+ return wait;
+ }
+
+ }
+
+ private SortedMap<TServerInstance,TabletServerStatus> gatherTableInformation() {
+ long start = System.currentTimeMillis();
+ SortedMap<TServerInstance,TabletServerStatus> result = new TreeMap<TServerInstance,TabletServerStatus>();
+ Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
+ for (TServerInstance server : currentServers) {
+ try {
+ Thread t = Thread.currentThread();
+ String oldName = t.getName();
+ try {
+ t.setName("Getting status from " + server);
+ TServerConnection connection = tserverSet.getConnection(server);
+ if (connection == null)
+ throw new IOException("No connection to " + server);
+ TabletServerStatus status = connection.getTableMap(false);
+ result.put(server, status);
+ } finally {
+ t.setName(oldName);
+ }
+ } catch (Exception ex) {
+ log.error("unable to get tablet server status " + server + " " + ex.toString());
+ log.debug("unable to get tablet server status " + server, ex);
+ if (badServers.get(server).incrementAndGet() > MAX_BAD_STATUS_COUNT) {
+ log.warn("attempting to stop " + server);
+ try {
+ TServerConnection connection = tserverSet.getConnection(server);
+ if (connection != null)
+ connection.halt(masterLock);
+ } catch (TTransportException e) {
+ // ignore: it's probably down
+ } catch (Exception e) {
+ log.info("error talking to troublesome tablet server ", e);
+ }
+ badServers.remove(server);
+ tserverSet.remove(server);
+ }
+ }
+ }
+ synchronized (badServers) {
+ badServers.keySet().retainAll(currentServers);
+ badServers.keySet().removeAll(result.keySet());
+ }
+ log.debug(String.format("Finished gathering information from %d servers in %.2f seconds", result.size(), (System.currentTimeMillis() - start) / 1000.));
+ return result;
+ }
+
+ public void run() throws IOException, InterruptedException, KeeperException {
+ final String zroot = ZooUtil.getRoot(instance);
+
+ getMasterLock(zroot + Constants.ZMASTER_LOCK);
+
+ recoveryManager = new RecoveryManager(this);
+
+ TableManager.getInstance().addObserver(this);
+
+ StatusThread statusThread = new StatusThread();
+ statusThread.start();
+
+ MigrationCleanupThread migrationCleanupThread = new MigrationCleanupThread();
+ migrationCleanupThread.start();
+
+ tserverSet.startListeningForTabletServerChanges();
+
+ ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ nextEvent.event("Noticed recovery changes", event.getType());
+ try {
+ // watcher only fires once, add it back
+ ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, this);
+ } catch (Exception e) {
+ log.error("Failed to add log recovery watcher back", e);
+ }
+ }
+ });
+
+ Credentials systemCreds = SystemCredentials.get();
+ watchers.add(new TabletGroupWatcher(this, new MetaDataStateStore(instance, systemCreds, this), null));
+ watchers.add(new TabletGroupWatcher(this, new RootTabletStateStore(instance, systemCreds, this), watchers.get(0)));
+ watchers.add(new TabletGroupWatcher(this, new ZooTabletStateStore(new ZooStore(zroot)), watchers.get(1)));
+ for (TabletGroupWatcher watcher : watchers) {
+ watcher.start();
+ }
+
+ // Once we are sure the upgrade is complete, we can safely allow fate use.
+ waitForMetadataUpgrade.await();
+
+ try {
+ final AgeOffStore<Master> store = new AgeOffStore<Master>(new org.apache.accumulo.fate.ZooStore<Master>(ZooUtil.getRoot(instance) + Constants.ZFATE,
+ ZooReaderWriter.getInstance()), 1000 * 60 * 60 * 8);
+
+ int threads = this.getConfiguration().getConfiguration().getCount(Property.MASTER_FATE_THREADPOOL_SIZE);
+
+ fate = new Fate<Master>(this, store, threads);
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+
+ @Override
+ public void run() {
+ store.ageOff();
+ }
+ }, 63000, 63000);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+
+ Processor<Iface> processor = new Processor<Iface>(RpcWrapper.service(new MasterClientServiceHandler(this)));
+ ServerAddress sa = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_CLIENTPORT, processor, "Master",
+ "Master Client Service Handler", null, Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
+ clientService = sa.server;
+ String address = sa.address.toString();
+ log.info("Setting master lock data to " + address);
+ masterLock.replaceLockData(address.getBytes());
+
+ while (!clientService.isServing()) {
+ UtilWaitThread.sleep(100);
+ }
+ while (clientService.isServing()) {
+ UtilWaitThread.sleep(500);
+ }
+ log.info("Shutting down fate.");
+ fate.shutdown();
+
+ final long deadline = System.currentTimeMillis() + MAX_CLEANUP_WAIT_TIME;
+ statusThread.join(remaining(deadline));
+
+ // quit, even if the tablet servers somehow jam up and the watchers
+ // don't stop
+ for (TabletGroupWatcher watcher : watchers) {
+ watcher.join(remaining(deadline));
+ }
+ log.info("exiting");
+ }
+
+ private long remaining(long deadline) {
+ return Math.max(1, deadline - System.currentTimeMillis());
+ }
+
+ public ZooLock getMasterLock() {
+ return masterLock;
+ }
+
+ private static class MasterLockWatcher implements ZooLock.AsyncLockWatcher {
+
+ boolean acquiredLock = false;
+ boolean failedToAcquireLock = false;
+
+ @Override
+ public void lostLock(LockLossReason reason) {
+ Halt.halt("Master lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
+ }
+
+ @Override
+ public void unableToMonitorLockNode(final Throwable e) {
+ Halt.halt(-1, new Runnable() {
+ @Override
+ public void run() {
+ log.fatal("No longer able to monitor master lock node", e);
+ }
+ });
+
+ }
+
+ @Override
+ public synchronized void acquiredLock() {
+ log.debug("Acquired master lock");
+
+ if (acquiredLock || failedToAcquireLock) {
+ Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ acquiredLock = true;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void failedToAcquireLock(Exception e) {
+ log.warn("Failed to get master lock " + e);
+
+ if (acquiredLock) {
+ Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ failedToAcquireLock = true;
+ notifyAll();
+ }
+
+ public synchronized void waitForChange() {
+ while (!acquiredLock && !failedToAcquireLock) {
+ try {
+ wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+
+ private void getMasterLock(final String zMasterLoc) throws KeeperException, InterruptedException {
+ log.info("trying to get master lock");
+
+ final String masterClientAddress = hostname + ":" + getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
+
+ while (true) {
+
+ MasterLockWatcher masterLockWatcher = new MasterLockWatcher();
+ masterLock = new ZooLock(zMasterLoc);
+ masterLock.lockAsync(masterLockWatcher, masterClientAddress.getBytes());
+
+ masterLockWatcher.waitForChange();
+
+ if (masterLockWatcher.acquiredLock) {
+ break;
+ }
+
+ if (!masterLockWatcher.failedToAcquireLock) {
+ throw new IllegalStateException("master lock in unknown state");
+ }
+
+ masterLock.tryToCancelAsyncLockOrUnlock();
+
+ UtilWaitThread.sleep(TIME_TO_WAIT_BETWEEN_LOCK_CHECKS);
+ }
+
+ setMasterState(MasterState.HAVE_LOCK);
+ }
+
+ public static void main(String[] args) throws Exception {
+ try {
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+
+ ServerOpts opts = new ServerOpts();
+ final String app = "master";
+ opts.parseArgs(app, args);
+ String hostname = opts.getAddress();
+ Accumulo.setupLogging(app);
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ VolumeManager fs = VolumeManagerImpl.get();
+ Accumulo.init(fs, conf, app);
+ Master master = new Master(conf, fs, hostname);
+ Accumulo.enableTracing(hostname, app);
+ master.run();
+ } catch (Exception ex) {
+ log.error("Unexpected exception, exiting", ex);
+ System.exit(1);
+ }
+ }
+
+ @Override
+ public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ DeadServerList obit = new DeadServerList(ZooUtil.getRoot(instance) + Constants.ZDEADTSERVERS);
+ if (added.size() > 0) {
+ log.info("New servers: " + added);
+ for (TServerInstance up : added)
+ obit.delete(up.hostPort());
+ }
+ for (TServerInstance dead : deleted) {
+ String cause = "unexpected failure";
+ if (serversToShutdown.contains(dead))
+ cause = "clean shutdown"; // maybe an incorrect assumption
+ if (!getMasterGoalState().equals(MasterGoalState.CLEAN_STOP))
+ obit.post(dead.hostPort(), cause);
+ }
+
+ Set<TServerInstance> unexpected = new HashSet<TServerInstance>(deleted);
+ unexpected.removeAll(this.serversToShutdown);
+ if (unexpected.size() > 0) {
+ if (stillMaster() && !getMasterGoalState().equals(MasterGoalState.CLEAN_STOP)) {
+ log.warn("Lost servers " + unexpected);
+ }
+ }
+ serversToShutdown.removeAll(deleted);
+ badServers.keySet().removeAll(deleted);
+ // clear out any bad server with the same host/port as a new server
+ synchronized (badServers) {
+ cleanListByHostAndPort(badServers.keySet(), deleted, added);
+ }
+ synchronized (serversToShutdown) {
+ cleanListByHostAndPort(serversToShutdown, deleted, added);
+ }
+
+ synchronized (migrations) {
+ Iterator<Entry<KeyExtent,TServerInstance>> iter = migrations.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<KeyExtent,TServerInstance> entry = iter.next();
+ if (deleted.contains(entry.getValue())) {
+ log.info("Canceling migration of " + entry.getKey() + " to " + entry.getValue());
+ iter.remove();
+ }
+ }
+ }
+ nextEvent.event("There are now %d tablet servers", current.size());
+ }
+
+ private static void cleanListByHostAndPort(Collection<TServerInstance> badServers, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ Iterator<TServerInstance> badIter = badServers.iterator();
+ while (badIter.hasNext()) {
+ TServerInstance bad = badIter.next();
+ for (TServerInstance add : added) {
+ if (bad.hostPort().equals(add.hostPort())) {
+ badIter.remove();
+ break;
+ }
+ }
+ for (TServerInstance del : deleted) {
+ if (bad.hostPort().equals(del.hostPort())) {
+ badIter.remove();
+ break;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void stateChanged(String tableId, TableState state) {
+ nextEvent.event("Table state in zookeeper changed for %s to %s", tableId, state);
+ if (TableState.OFFLINE == state) {
+ clearMigrations(tableId);
+ }
+ }
+
+ @Override
+ public void initialize(Map<String,TableState> tableIdToStateMap) {}
+
+ @Override
+ public void sessionExpired() {}
+
+ @Override
+ public Set<String> onlineTables() {
+ Set<String> result = new HashSet<String>();
+ if (getMasterState() != MasterState.NORMAL) {
+ if (getMasterState() != MasterState.UNLOAD_METADATA_TABLETS)
+ result.add(MetadataTable.ID);
+ if (getMasterState() != MasterState.UNLOAD_ROOT_TABLET)
+ result.add(RootTable.ID);
+ return result;
+ }
+ TableManager manager = TableManager.getInstance();
+
+ for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
+ TableState state = manager.getTableState(tableId);
+ if (state != null) {
+ if (state == TableState.ONLINE)
+ result.add(tableId);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public Set<TServerInstance> onlineTabletServers() {
+ return tserverSet.getCurrentServers();
+ }
+
+ @Override
+ public Collection<MergeInfo> merges() {
+ List<MergeInfo> result = new ArrayList<MergeInfo>();
+ for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
+ result.add(getMergeInfo(new Text(tableId)));
+ }
+ return result;
+ }
+
+ // recovers state from the persistent transaction to shutdown a server
+ public void shutdownTServer(TServerInstance server) {
+ nextEvent.event("Tablet Server shutdown requested for %s", server);
+ serversToShutdown.add(server);
+ }
+
+ public EventCoordinator getEventCoordinator() {
+ return nextEvent;
+ }
+
+ public Instance getInstance() {
+ return this.instance;
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ public ServerConfiguration getConfiguration() {
+ return serverConfig;
+ }
+
+ public VolumeManager getFileSystem() {
+ return this.fs;
+ }
+
+ public void assignedTablet(KeyExtent extent) {
+ if (extent.isMeta()) {
+ if (getMasterState().equals(MasterState.UNLOAD_ROOT_TABLET)) {
+ setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
+ }
+ }
+ if (extent.isRootTablet()) {
+ // probably too late, but try anyhow
+ if (getMasterState().equals(MasterState.STOP)) {
+ setMasterState(MasterState.UNLOAD_ROOT_TABLET);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index 0c2e1f0,0000000..5e478cb
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@@ -1,190 -1,0 +1,192 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.recovery;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.log.SortedLogState;
+import org.apache.accumulo.server.master.recovery.HadoopLogCloser;
+import org.apache.accumulo.server.master.recovery.LogCloser;
+import org.apache.accumulo.server.master.recovery.RecoveryPath;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class RecoveryManager {
+
+ private static Logger log = Logger.getLogger(RecoveryManager.class);
+
+ private Map<String,Long> recoveryDelay = new HashMap<String,Long>();
+ private Set<String> closeTasksQueued = new HashSet<String>();
+ private Set<String> sortsQueued = new HashSet<String>();
+ private ScheduledExecutorService executor;
+ private Master master;
+ private ZooCache zooCache;
+
+ public RecoveryManager(Master master) {
+ this.master = master;
+ executor = Executors.newScheduledThreadPool(4, new NamingThreadFactory("Walog sort starter "));
+ zooCache = new ZooCache();
+ try {
+ List<String> workIDs = new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).getWorkQueued();
+ sortsQueued.addAll(workIDs);
+ } catch (Exception e) {
+ log.warn(e, e);
+ }
+ }
+
+ private class LogSortTask implements Runnable {
+ private String source;
+ private String destination;
+ private String sortId;
+ private LogCloser closer;
+
+ public LogSortTask(LogCloser closer, String source, String destination, String sortId) {
+ this.closer = closer;
+ this.source = source;
+ this.destination = destination;
+ this.sortId = sortId;
+ }
+
+ @Override
+ public void run() {
+ boolean rescheduled = false;
+ try {
+
+ long time = closer.close(master.getConfiguration().getConfiguration(), master.getFileSystem(), new Path(source));
+
+ if (time > 0) {
+ executor.schedule(this, time, TimeUnit.MILLISECONDS);
+ rescheduled = true;
+ } else {
+ initiateSort(sortId, source, destination);
+ }
+ } catch (FileNotFoundException e) {
+ log.debug("Unable to initate log sort for " + source + ": " + e);
+ } catch (Exception e) {
+ log.warn("Failed to initiate log sort " + source, e);
+ } finally {
+ if (!rescheduled) {
+ synchronized (RecoveryManager.this) {
+ closeTasksQueued.remove(sortId);
+ }
+ }
+ }
+ }
+
+ }
+
+ private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
+ String work = source + "|" + destination;
- new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(Constants.UTF8));
++ new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(UTF_8));
+
+ synchronized (this) {
+ sortsQueued.add(sortId);
+ }
+
+ final String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId;
+ log.info("Created zookeeper entry " + path + " with data " + work);
+ }
+
+ public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>> walogs) throws IOException {
+ boolean recoveryNeeded = false;
+
+ for (Collection<String> logs : walogs) {
+ for (String walog : logs) {
+
+ String switchedWalog = VolumeUtil.switchVolume(walog, FileType.WAL, ServerConstants.getVolumeReplacements());
+ if (switchedWalog != null) {
+ // replaces the volume used for sorting, but do not change entry in metadata table. When the tablet loads it will change the metadata table entry. If
+ // the tablet has the same replacement config, then it will find the sorted log.
+ log.info("Volume replaced " + walog + " -> " + switchedWalog);
+ walog = switchedWalog;
+ }
+
+ String parts[] = walog.split("/");
+ String sortId = parts[parts.length - 1];
+ String filename = master.getFileSystem().getFullPath(FileType.WAL, walog).toString();
+ String dest = RecoveryPath.getRecoveryPath(master.getFileSystem(), new Path(filename)).toString();
+ log.debug("Recovering " + filename + " to " + dest);
+
+ boolean sortQueued;
+ synchronized (this) {
+ sortQueued = sortsQueued.contains(sortId);
+ }
+
+ if (sortQueued && zooCache.get(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId) == null) {
+ synchronized (this) {
+ sortsQueued.remove(sortId);
+ }
+ }
+
+ if (master.getFileSystem().exists(SortedLogState.getFinishedMarkerPath(dest))) {
+ synchronized (this) {
+ closeTasksQueued.remove(sortId);
+ recoveryDelay.remove(sortId);
+ sortsQueued.remove(sortId);
+ }
+ continue;
+ }
+
+ recoveryNeeded = true;
+ synchronized (this) {
+ if (!closeTasksQueued.contains(sortId) && !sortsQueued.contains(sortId)) {
+ AccumuloConfiguration aconf = master.getConfiguration().getConfiguration();
+ LogCloser closer = aconf.instantiateClassProperty(Property.MASTER_WALOG_CLOSER_IMPLEMETATION, LogCloser.class, new HadoopLogCloser());
+ Long delay = recoveryDelay.get(sortId);
+ if (delay == null) {
+ delay = master.getSystemConfiguration().getTimeInMillis(Property.MASTER_RECOVERY_DELAY);
+ } else {
+ delay = Math.min(2 * delay, 1000 * 60 * 5l);
+ }
+
+ log.info("Starting recovery of " + filename + " (in : " + (delay / 1000) + "s), tablet " + extent + " holds a reference");
+
+ executor.schedule(new LogSortTask(closer, filename, dest, sortId), delay, TimeUnit.MILLISECONDS);
+ closeTasksQueued.add(sortId);
+ recoveryDelay.put(sortId, delay);
+ }
+ }
+ }
+ }
+ return recoveryNeeded;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
index f981bae,0000000..a674557
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
@@@ -1,49 -1,0 +1,51 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+public class SetGoalState {
+
+ /**
+ * Utility program that will change the goal state for the master from the command line.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length != 1 || MasterGoalState.valueOf(args[0]) == null) {
+ System.err.println("Usage: accumulo " + SetGoalState.class.getName() + " [NORMAL|SAFE_MODE|CLEAN_STOP]");
+ System.exit(-1);
+ }
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ Accumulo.waitForZookeeperAndHdfs(fs);
- ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(Constants.UTF8),
++ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(UTF_8),
+ NodeExistsPolicy.OVERWRITE);
+ }
+
+}
[28/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index 665b132,0000000..4975428
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -1,356 -1,0 +1,357 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+
+import jline.console.ConsoleReader;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.shell.Shell;
+
+public class ShellServlet extends BasicServlet {
+ private static final long serialVersionUID = 1L;
+ private Map<String,ShellExecutionThread> userShells = new HashMap<String,ShellExecutionThread>();
+ private ExecutorService service = Executors.newCachedThreadPool();
+
+ public static final String CSRF_KEY = "csrf_token";
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "Shell";
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws IOException {
+ HttpSession session = req.getSession(true);
+ final String CSRF_TOKEN;
+ if (null == session.getAttribute(CSRF_KEY)) {
+ // No token, make one
+ CSRF_TOKEN = UUID.randomUUID().toString();
+ session.setAttribute(CSRF_KEY, CSRF_TOKEN);
+ } else {
+ // Pull the token out of the session
+ CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
+ if (null == CSRF_TOKEN) {
+ throw new RuntimeException("No valid CSRF token exists in session");
+ }
+ }
+
+ String user = (String) session.getAttribute("user");
+ if (user == null) {
+ // user attribute is null, check to see if username and password are passed as parameters
+ user = req.getParameter("user");
+ String pass = req.getParameter("pass");
+ String mock = req.getParameter("mock");
+ if (user == null || pass == null) {
+ // username or password are null, re-authenticate
+ sb.append(authenticationForm(req.getRequestURI(), CSRF_TOKEN));
+ return;
+ }
+ try {
+ // get a new shell for this user
+ ShellExecutionThread shellThread = new ShellExecutionThread(user, pass, mock);
+ service.submit(shellThread);
+ userShells.put(session.getId(), shellThread);
+ } catch (IOException e) {
+ // error validating user, reauthenticate
+ sb.append("<div id='loginError'>Invalid user/password</div>" + authenticationForm(req.getRequestURI(), CSRF_TOKEN));
+ return;
+ }
+ session.setAttribute("user", user);
+ }
+ if (!userShells.containsKey(session.getId())) {
+ // no existing shell for this user, re-authenticate
+ sb.append(authenticationForm(req.getRequestURI(), UUID.randomUUID().toString()));
+ return;
+ }
+
+ ShellExecutionThread shellThread = userShells.get(session.getId());
+ shellThread.getOutput();
+ shellThread.printInfo();
+ sb.append("<div id='shell'>\n");
+ sb.append("<pre id='shellResponse'>").append(shellThread.getOutput()).append("</pre>\n");
+ sb.append("<form><span id='shellPrompt'>").append(shellThread.getPrompt());
+ sb.append("</span><input type='text' name='cmd' id='cmd' onkeydown='return handleKeyDown(event.keyCode);'>\n");
+ sb.append("</form>\n</div>\n");
+ sb.append("<script type='text/javascript'>\n");
+ sb.append("var url = '").append(req.getRequestURL().toString()).append("';\n");
+ sb.append("var xmlhttp = new XMLHttpRequest();\n");
+ sb.append("var hsize = 1000;\n");
+ sb.append("var hindex = 0;\n");
+ sb.append("var history = new Array();\n");
+ sb.append("\n");
+ sb.append("function handleKeyDown(keyCode) {\n");
+ sb.append(" if (keyCode==13) {\n");
+ sb.append(" submitCmd(document.getElementById('cmd').value);\n");
+ sb.append(" hindex = history.length;\n");
+ sb.append(" return false;\n");
+ sb.append(" } else if (keyCode==38) {\n");
+ sb.append(" hindex = hindex==0 ? history.length : hindex - 1;\n");
+ sb.append(" if (hindex == history.length)\n");
+ sb.append(" document.getElementById('cmd').value = '';\n");
+ sb.append(" else\n");
+ sb.append(" document.getElementById('cmd').value = history[hindex];\n");
+ sb.append(" return false;\n");
+ sb.append(" } else if (keyCode==40) {\n");
+ sb.append(" hindex = hindex==history.length ? history.length : hindex + 1;\n");
+ sb.append(" if (hindex == history.length)\n");
+ sb.append(" document.getElementById('cmd').value = '';\n");
+ sb.append(" else\n");
+ sb.append(" document.getElementById('cmd').value = history[hindex];\n");
+ sb.append(" return false;\n");
+ sb.append(" }\n");
+ sb.append(" return true;\n");
+ sb.append("}\n");
+ sb.append("\n");
+ sb.append("function submitCmd(cmd) {\n");
+ sb.append(" if (cmd=='history') {\n");
+ sb.append(" document.getElementById('shellResponse').innerHTML += document.getElementById('shellPrompt').innerHTML+cmd+'\\n';\n");
+ sb.append(" document.getElementById('shellResponse').innerHTML += history.join('\\n');\n");
+ sb.append(" return\n");
+ sb.append(" }\n");
+ sb.append(" xmlhttp.open('POST',url+'?cmd='+cmd+'&'+'").append(CSRF_KEY).append("=").append(CSRF_TOKEN).append("',false);\n");
+ sb.append(" xmlhttp.send();\n");
+ sb.append(" var text = xmlhttp.responseText;\n");
+ sb.append(" var index = text.lastIndexOf('\\n');\n");
+ sb.append(" if (index >= 0) {\n");
+ sb.append(" if (index > 0 && document.getElementById('cmd').type == 'text') {\n");
+ sb.append(" if (history.length == hsize)\n");
+ sb.append(" history.shift()\n");
+ sb.append(" history.push(cmd)\n");
+ sb.append(" }\n");
+ sb.append(" if (text.charAt(text.length-1)=='*') {\n");
+ sb.append(" document.getElementById('cmd').type = 'password';\n");
+ sb.append(" text = text.substring(0,xmlhttp.responseText.length-2);\n");
+ sb.append(" } else {\n");
+ sb.append(" document.getElementById('cmd').type = 'text';\n");
+ sb.append(" }\n");
+ sb.append(" document.getElementById('shellResponse').innerHTML += text.substring(0,index+1);\n");
+ sb.append(" document.getElementById('shellPrompt').innerHTML = text.substring(index+1);\n");
+ sb.append(" document.getElementById('cmd').value = '';\n");
+ sb.append(" document.getElementById('shell').scrollTop = document.getElementById('cmd').offsetTop;\n");
+ sb.append(" } else {\n");
+ sb.append(" window.location = url;\n");
+ sb.append(" }\n");
+ sb.append("}\n");
+ sb.append("</script>\n");
+ sb.append("<script type='text/javascript'>window.onload = function() { document.getElementById('cmd').select(); }</script>\n");
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ final HttpSession session = req.getSession(true);
+ String user = (String) session.getAttribute("user");
+ if (user == null || !userShells.containsKey(session.getId())) {
+ // no existing shell for user, re-authenticate
+ doGet(req, resp);
+ return;
+ }
+ final String CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
+ if (null == CSRF_TOKEN) {
+ // no csrf token, need to re-auth
+ doGet(req, resp);
+ }
+ ShellExecutionThread shellThread = userShells.get(session.getId());
+ String cmd = req.getParameter("cmd");
+ if (cmd == null) {
+ // the command is null, just print prompt
+ resp.getWriter().append(shellThread.getPrompt());
+ resp.getWriter().flush();
+ return;
+ }
+ shellThread.addInputString(cmd);
+ shellThread.waitUntilReady();
+ if (shellThread.isDone()) {
+ // the command was exit, invalidate session
+ userShells.remove(session.getId());
+ session.invalidate();
+ return;
+ }
+ // get the shell's output
+ StringBuilder sb = new StringBuilder();
+ sb.append(shellThread.getOutput().replace("<", "<").replace(">", ">"));
+ if (sb.length() == 0 || !(sb.charAt(sb.length() - 1) == '\n'))
+ sb.append("\n");
+ // check if shell is waiting for input
+ if (!shellThread.isWaitingForInput())
+ sb.append(shellThread.getPrompt());
+ // check if shell is waiting for password input
+ if (shellThread.isMasking())
+ sb.append("*");
+ resp.getWriter().append(sb.toString());
+ resp.getWriter().flush();
+ }
+
+ private String authenticationForm(String requestURI, String csrfToken) {
+ return "<div id='login'><form method=POST action='" + requestURI + "'>"
+ + "<table><tr><td>Mock: </td><td><input type='checkbox' name='mock' value='mock'></td></tr>"
+ + "<tr><td>Username: </td><td><input type='text' name='user'></td></tr>"
+ + "<tr><td>Password: </td><td><input type='password' name='pass'></td><td>"
+ + "<input type='hidden' name='" + CSRF_KEY + "' value='" + csrfToken + "'/><input type='submit' value='Enter'></td></tr></table></form></div>";
+ }
+
+ private static class StringBuilderOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static class ShellExecutionThread extends InputStream implements Runnable {
+ private Shell shell;
+ StringBuilderOutputStream output;
+ private String cmd;
+ private int cmdIndex;
+ private boolean done;
+ private boolean readWait;
+
+ private ShellExecutionThread(String username, String password, String mock) throws IOException {
+ this.done = false;
+ this.cmd = null;
+ this.cmdIndex = 0;
+ this.readWait = false;
+ this.output = new StringBuilderOutputStream();
+ ConsoleReader reader = new ConsoleReader(this, output);
- this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, Constants.UTF8)));
++ this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, UTF_8)));
+ shell.setLogErrorsToConsole();
+ if (mock != null) {
+ if (shell.config("--fake", "-u", username, "-p", password))
+ throw new IOException("mock shell config error");
+ } else if (shell.config("-u", username, "-p", password)) {
+ throw new IOException("shell config error");
+ }
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ if (cmd == null) {
+ readWait = true;
+ this.notifyAll();
+ }
+ while (cmd == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {}
+ }
+ readWait = false;
+ int c;
+ if (cmdIndex == cmd.length())
+ c = '\n';
+ else
+ c = cmd.charAt(cmdIndex);
+ cmdIndex++;
+ if (cmdIndex > cmd.length()) {
+ cmd = null;
+ cmdIndex = 0;
+ this.notifyAll();
+ }
+ return c;
+ }
+
+ @Override
+ public synchronized void run() {
+ Thread.currentThread().setName("shell thread");
+ while (!shell.hasExited()) {
+ while (cmd == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {}
+ }
+ String tcmd = cmd;
+ cmd = null;
+ cmdIndex = 0;
+ try {
+ shell.execCommand(tcmd, false, true);
+ } catch (IOException e) {}
+ this.notifyAll();
+ }
+ done = true;
+ this.notifyAll();
+ }
+
+ public synchronized void addInputString(String s) {
+ if (done)
+ throw new IllegalStateException("adding string to exited shell");
+ if (cmd == null) {
+ cmd = s;
+ } else {
+ throw new IllegalStateException("adding string to shell not waiting for input");
+ }
+ this.notifyAll();
+ }
+
+ public synchronized void waitUntilReady() {
+ while (cmd != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+
+ public synchronized String getOutput() {
+ String s = output.get();
+ output.clear();
+ return s;
+ }
+
+ public String getPrompt() {
+ return shell.getDefaultPrompt();
+ }
+
+ public void printInfo() throws IOException {
+ shell.printInfo();
+ }
+
+ public boolean isMasking() {
+ return shell.isMasking();
+ }
+
+ public synchronized boolean isWaitingForInput() {
+ return readWait;
+ }
+
+ public boolean isDone() {
+ return done;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
index 0cbf957,0000000..3cd635e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
@@@ -1,179 -1,0 +1,180 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.DeadServer;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.state.TabletServerState;
+import org.apache.accumulo.server.util.TableInfoUtil;
+
+public class XMLServlet extends BasicServlet {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "XML Report";
+ }
+
+ @Override
+ protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
- resp.setContentType("text/xml;charset=" + Constants.UTF8.name());
- sb.append("<?xml version=\"1.0\" encoding=\"" + Constants.UTF8.name() + "\"?>\n");
++ resp.setContentType("text/xml;charset=" + UTF_8.name());
++ sb.append("<?xml version=\"1.0\" encoding=\"" + UTF_8.name() + "\"?>\n");
+ sb.append("<stats>\n");
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
+ double totalIngest = 0.;
+ double totalQuery = 0.;
+ double disk = 0.0;
+ long totalEntries = 0L;
+
+ sb.append("\n<servers>\n");
+ if (Monitor.getMmi() == null || Monitor.getMmi().tableMap == null) {
+ sb.append("</servers>\n");
+ return;
+ }
+ SortedMap<String,TableInfo> tableStats = new TreeMap<String,TableInfo>(Monitor.getMmi().tableMap);
+
+ for (TabletServerStatus status : Monitor.getMmi().tServerInfo) {
+
+ sb.append("\n<server id='").append(status.name).append("'>\n");
+ sb.append("<hostname>").append(TServerLinkType.displayName(status.name)).append("</hostname>");
+ sb.append("<lastContact>").append(System.currentTimeMillis() - status.lastContact).append("</lastContact>\n");
+ sb.append("<osload>").append(status.osLoad).append("</osload>\n");
+
+ TableInfo summary = TableInfoUtil.summarizeTableStats(status);
+ sb.append("<compactions>\n");
+ sb.append("<major>").append("<running>").append(summary.majors.running).append("</running>").append("<queued>").append(summary.majors.queued)
+ .append("</queued>").append("</major>\n");
+ sb.append("<minor>").append("<running>").append(summary.minors.running).append("</running>").append("<queued>").append(summary.minors.queued)
+ .append("</queued>").append("</minor>\n");
+ sb.append("</compactions>\n");
+
+ sb.append("<tablets>").append(summary.tablets).append("</tablets>\n");
+
+ sb.append("<ingest>").append(summary.ingestRate).append("</ingest>\n");
+ sb.append("<query>").append(summary.queryRate).append("</query>\n");
+ sb.append("<ingestMB>").append(summary.ingestByteRate / 1000000.0).append("</ingestMB>\n");
+ sb.append("<queryMB>").append(summary.queryByteRate / 1000000.0).append("</queryMB>\n");
+ sb.append("<scans>").append(summary.scans.running + summary.scans.queued).append("</scans>");
+ sb.append("<scansessions>").append(Monitor.getLookupRate()).append("</scansessions>\n");
+ sb.append("<holdtime>").append(status.holdTime).append("</holdtime>\n");
+ totalIngest += summary.ingestRate;
+ totalQuery += summary.queryRate;
+ totalEntries += summary.recs;
+ sb.append("</server>\n");
+ }
+ sb.append("\n</servers>\n");
+
+ sb.append("\n<masterGoalState>" + Monitor.getMmi().goalState + "</masterGoalState>\n");
+ sb.append("\n<masterState>" + Monitor.getMmi().state + "</masterState>\n");
+
+ sb.append("\n<badTabletServers>\n");
+ for (Entry<String,Byte> entry : Monitor.getMmi().badTServers.entrySet()) {
+ sb.append(String.format("<badTabletServer id='%s' status='%s'/>\n", entry.getKey(), TabletServerState.getStateById(entry.getValue())));
+ }
+ sb.append("\n</badTabletServers>\n");
+
+ sb.append("\n<tabletServersShuttingDown>\n");
+ for (String server : Monitor.getMmi().serversShuttingDown) {
+ sb.append(String.format("<server id='%s'/>\n", server));
+ }
+ sb.append("\n</tabletServersShuttingDown>\n");
+
+ sb.append(String.format("\n<unassignedTablets>%d</unassignedTablets>\n", Monitor.getMmi().unassignedTablets));
+
+ sb.append("\n<deadTabletServers>\n");
+ for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
+ sb.append(String.format("<deadTabletServer id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
+ }
+ sb.append("\n</deadTabletServers>\n");
+
+ sb.append("\n<deadLoggers>\n");
+ for (DeadServer dead : Monitor.getMmi().deadTabletServers) {
+ sb.append(String.format("<deadLogger id='%s' lastChange='%d' status='%s'/>\n", dead.server, dead.lastStatus, dead.status));
+ }
+ sb.append("\n</deadLoggers>\n");
+
+ sb.append("\n<tables>\n");
+ Instance instance = HdfsZooInstance.getInstance();
+ for (Entry<String,TableInfo> entry : tableStats.entrySet()) {
+ TableInfo tableInfo = entry.getValue();
+
+ sb.append("\n<table>\n");
+ String tableId = entry.getKey();
+ String tableName = "unknown";
+ String tableState = "unknown";
+ try {
+ tableName = Tables.getTableName(instance, tableId);
+ tableState = Tables.getTableState(instance, tableId).toString();
+ } catch (Exception ex) {
+ log.warn(ex, ex);
+ }
+ sb.append("<tablename>").append(tableName).append("</tablename>\n");
+ sb.append("<tableId>").append(tableId).append("</tableId>\n");
+ sb.append("<tableState>").append(tableState).append("</tableState>\n");
+ sb.append("<tablets>").append(tableInfo.tablets).append("</tablets>\n");
+ sb.append("<onlineTablets>").append(tableInfo.onlineTablets).append("</onlineTablets>\n");
+ sb.append("<recs>").append(tableInfo.recs).append("</recs>\n");
+ sb.append("<recsInMemory>").append(tableInfo.recsInMemory).append("</recsInMemory>\n");
+ sb.append("<ingest>").append(tableInfo.ingestRate).append("</ingest>\n");
+ sb.append("<ingestByteRate>").append(tableInfo.ingestByteRate).append("</ingestByteRate>\n");
+ sb.append("<query>").append(tableInfo.queryRate).append("</query>\n");
+ sb.append("<queryByteRate>").append(tableInfo.queryRate).append("</queryByteRate>\n");
+ int running = 0;
+ int queued = 0;
+ Compacting compacting = entry.getValue().majors;
+ if (compacting != null) {
+ running = compacting.running;
+ queued = compacting.queued;
+ }
+ sb.append("<majorCompactions>").append("<running>").append(running).append("</running>").append("<queued>").append(queued).append("</queued>")
+ .append("</majorCompactions>\n");
+ sb.append("</table>\n");
+ }
+ sb.append("\n</tables>\n");
+
+ sb.append("\n<totals>\n");
+ sb.append("<ingestrate>").append(totalIngest).append("</ingestrate>\n");
+ sb.append("<queryrate>").append(totalQuery).append("</queryrate>\n");
+ sb.append("<diskrate>").append(disk).append("</diskrate>\n");
+ sb.append("<numentries>").append(totalEntries).append("</numentries>\n");
+ sb.append("</totals>\n");
+ }
+
+ @Override
+ protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
+ sb.append("\n</stats>\n");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 90bb0d4,0000000..1c464d4
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@@ -1,105 -1,0 +1,106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets.trace;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Date;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.servlet.http.HttpServletRequest;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.servlets.BasicServlet;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
+abstract class Basic extends BasicServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ public static String getStringParameter(HttpServletRequest req, String name, String defaultValue) {
+ String result = req.getParameter(name);
+ if (result == null) {
+ return defaultValue;
+ }
+ return result;
+ }
+
+ public static int getIntParameter(HttpServletRequest req, String name, int defaultMinutes) {
+ String valueString = req.getParameter(name);
+ if (valueString == null)
+ return defaultMinutes;
+ int result = 0;
+ try {
+ result = Integer.parseInt(valueString);
+ } catch (NumberFormatException ex) {
+ return defaultMinutes;
+ }
+ return result;
+ }
+
+ public static String dateString(long millis) {
+ return TraceFormatter.formatDate(new Date(millis));
+ }
+
+ protected Scanner getScanner(StringBuilder sb) throws AccumuloException, AccumuloSecurityException {
+ AccumuloConfiguration conf = Monitor.getSystemConfiguration();
+ String principal = conf.get(Property.TRACE_USER);
+ AuthenticationToken at;
+ Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+ if (loginMap.isEmpty()) {
+ Property p = Property.TRACE_PASSWORD;
- at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++ at = new PasswordToken(conf.get(p).getBytes(UTF_8));
+ } else {
+ Properties props = new Properties();
+ int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
+ for (Entry<String,String> entry : loginMap.entrySet()) {
+ props.put(entry.getKey().substring(prefixLength), entry.getValue());
+ }
+
+ AuthenticationToken token = Property.createInstanceFromPropertyName(conf, Property.TRACE_TOKEN_TYPE, AuthenticationToken.class, new PasswordToken());
+ token.init(props);
+ at = token;
+ }
+
+ String table = conf.get(Property.TRACE_TABLE);
+ try {
+ Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
+ if (!conn.tableOperations().exists(table)) {
+ return new NullScanner();
+ }
+ Scanner scanner = conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
+ return scanner;
+ } catch (AccumuloSecurityException ex) {
+ sb.append("<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
+ return null;
+ } catch (TableNotFoundException ex) {
+ return new NullScanner();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
----------------------------------------------------------------------
diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index e3ae0c8,0000000..6094911
mode 100644,000000..100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -1,324 -1,0 +1,326 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tracer;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.thrift.RemoteSpan;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TByteArrayOutputStream;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+
+public class TraceServer implements Watcher {
+
+ final private static Logger log = Logger.getLogger(TraceServer.class);
+ final private ServerConfiguration serverConfiguration;
+ final private TServer server;
+ final private AtomicReference<BatchWriter> writer;
+ final private Connector connector;
+ final String table;
+
+ private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) {
+ m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
+ }
+
+ static class ByteArrayTransport extends TTransport {
+ TByteArrayOutputStream out = new TByteArrayOutputStream();
+
+ @Override
+ public boolean isOpen() {
+ return true;
+ }
+
+ @Override
+ public void open() throws TTransportException {}
+
+ @Override
+ public void close() {}
+
+ @Override
+ public int read(byte[] buf, int off, int len) {
+ return 0;
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) throws TTransportException {
+ out.write(buf, off, len);
+ }
+
+ public byte[] get() {
+ return out.get();
+ }
+
+ public int len() {
+ return out.len();
+ }
+ }
+
+ class Receiver implements Iface {
+ @Override
+ public void span(RemoteSpan s) throws TException {
+ String idString = Long.toHexString(s.traceId);
+ String startString = Long.toHexString(s.start);
+ Mutation spanMutation = new Mutation(new Text(idString));
+ Mutation indexMutation = new Mutation(new Text("idx:" + s.svc + ":" + startString));
+ long diff = s.stop - s.start;
- indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(Constants.UTF8)));
++ indexMutation.put(new Text(s.description), new Text(s.sender), new Value((idString + ":" + Long.toHexString(diff)).getBytes(UTF_8)));
+ ByteArrayTransport transport = new ByteArrayTransport();
+ TCompactProtocol protocol = new TCompactProtocol(transport);
+ s.write(protocol);
+ String parentString = Long.toHexString(s.parentId);
+ if (s.parentId == Span.ROOT_SPAN_ID)
+ parentString = "";
+ put(spanMutation, "span", parentString + ":" + Long.toHexString(s.spanId), transport.get(), transport.len());
+ // Map the root span to time so we can look up traces by time
+ Mutation timeMutation = null;
+ if (s.parentId == Span.ROOT_SPAN_ID) {
+ timeMutation = new Mutation(new Text("start:" + startString));
+ put(timeMutation, "id", idString, transport.get(), transport.len());
+ }
+ try {
+ final BatchWriter writer = TraceServer.this.writer.get();
+ /*
+ * Check for null, because we expect spans to come in much faster than flush calls. In the case of failure, we'd rather avoid logging tons of NPEs.
+ */
+ if (null == writer) {
+ log.warn("writer is not ready; discarding span.");
+ return;
+ }
+ writer.addMutation(spanMutation);
+ writer.addMutation(indexMutation);
+ if (timeMutation != null)
+ writer.addMutation(timeMutation);
+ } catch (MutationsRejectedException exception) {
+ log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception);
+ if (log.isDebugEnabled()) {
+ log.debug("discarded span due to rejection of mutation: " + spanMutation, exception);
+ }
+ /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */
+ } catch (RuntimeException exception) {
+ log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception);
+ log.debug("unable to write mutation to table due to exception.", exception);
+ }
+ }
+
+ }
+
+ public TraceServer(ServerConfiguration serverConfiguration, String hostname) throws Exception {
+ this.serverConfiguration = serverConfiguration;
+ AccumuloConfiguration conf = serverConfiguration.getConfiguration();
+ table = conf.get(Property.TRACE_TABLE);
+ Connector connector = null;
+ while (true) {
+ try {
+ String principal = conf.get(Property.TRACE_USER);
+ AuthenticationToken at;
+ Map<String,String> loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+ if (loginMap.isEmpty()) {
+ Property p = Property.TRACE_PASSWORD;
- at = new PasswordToken(conf.get(p).getBytes(Constants.UTF8));
++ at = new PasswordToken(conf.get(p).getBytes(UTF_8));
+ } else {
+ Properties props = new Properties();
+ AuthenticationToken token = AccumuloVFSClassLoader.getClassLoader().loadClass(conf.get(Property.TRACE_TOKEN_TYPE)).asSubclass(AuthenticationToken.class)
+ .newInstance();
+
+ int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
+ for (Entry<String,String> entry : loginMap.entrySet()) {
+ props.put(entry.getKey().substring(prefixLength), entry.getValue());
+ }
+
+ token.init(props);
+
+ at = token;
+ }
+
+ connector = serverConfiguration.getInstance().getConnector(principal, at);
+ if (!connector.tableOperations().exists(table)) {
+ connector.tableOperations().create(table);
+ IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName());
+ AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
+ connector.tableOperations().attachIterator(table, setting);
+ }
+ connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
+ break;
+ } catch (Exception ex) {
+ log.info("Waiting to checking/create the trace table.", ex);
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ this.connector = connector;
+ // make sure we refer to the final variable from now on.
+ connector = null;
+
+ int port = conf.getPort(Property.TRACE_PORT);
+ final ServerSocket sock = ServerSocketChannel.open().socket();
+ sock.setReuseAddress(true);
+ sock.bind(new InetSocketAddress(hostname, port));
+ final TServerTransport transport = new TServerSocket(sock);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.processor(new Processor<Iface>(new Receiver()));
+ server = new TThreadPoolServer(options);
+ registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort());
+ writer = new AtomicReference<BatchWriter>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS)));
+ }
+
+ public void run() throws Exception {
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ flush();
+ }
+ }, 1000, 1000);
+ server.serve();
+ }
+
+ private void flush() {
+ try {
+ final BatchWriter writer = this.writer.get();
+ if (null != writer) {
+ writer.flush();
+ }
+ } catch (MutationsRejectedException exception) {
+ log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
+ log.debug("flushing traces failed due to exception", exception);
+ resetWriter();
+ /* XXX e.g. if the writer was closed between when we grabbed it and when we called flush. */
+ } catch (RuntimeException exception) {
+ log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
+ log.debug("flushing traces failed due to exception", exception);
+ resetWriter();
+ }
+ }
+
+ private void resetWriter() {
+ BatchWriter writer = null;
+ try {
+ writer = connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS));
+ } catch (Exception ex) {
+ log.warn("Unable to create a batch writer, will retry. Set log level to DEBUG to see stacktrace. cause: " + ex);
+ log.debug("batch writer creation failed with exception.", ex);
+ } finally {
+ /* Trade in the new writer (even if null) for the one we need to close. */
+ writer = this.writer.getAndSet(writer);
+ try {
+ if (null != writer) {
+ writer.close();
+ }
+ } catch (Exception ex) {
+ log.warn("Problem closing batch writer. Set log level to DEBUG to see stacktrace. cause: " + ex);
+ log.debug("batch writer close failed with exception", ex);
+ }
+ }
+ }
+
+ private void registerInZooKeeper(String name) throws Exception {
+ String root = ZooUtil.getRoot(serverConfiguration.getInstance()) + Constants.ZTRACERS;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(Constants.UTF8));
++ String path = zoo.putEphemeralSequential(root + "/trace-", name.getBytes(UTF_8));
+ zoo.exists(path, this);
+ }
+
+ public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+ ServerOpts opts = new ServerOpts();
+ final String app = "tracer";
+ opts.parseArgs(app, args);
+ Accumulo.setupLogging(app);
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ VolumeManager fs = VolumeManagerImpl.get();
+ Accumulo.init(fs, conf, app);
+ String hostname = opts.getAddress();
+ TraceServer server = new TraceServer(conf, hostname);
+ Accumulo.enableTracing(hostname, app);
+ server.run();
+ log.info("tracer stopping");
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
+ if (event.getState() == KeeperState.Expired) {
+ log.warn("Trace server lost zookeeper registration at " + event.getPath());
+ server.stop();
+ } else if (event.getType() == EventType.NodeDeleted) {
+ log.warn("Trace server zookeeper entry lost " + event.getPath());
+ server.stop();
+ }
+ if (event.getPath() != null) {
+ try {
+ if (ZooReaderWriter.getInstance().exists(event.getPath(), this))
+ return;
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ log.warn("Trace server unable to reset watch on zookeeper registration");
+ server.stop();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index 39bf81e,0000000..4d9a6c2
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@@ -1,75 -1,0 +1,76 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Copy failed bulk imports.
+ */
+public class BulkFailedCopyProcessor implements Processor {
+
+ private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
+
+ @Override
+ public Processor newProcessor() {
+ return new BulkFailedCopyProcessor();
+ }
+
+ @Override
+ public void process(String workID, byte[] data) {
+
- String paths[] = new String(data, Constants.UTF8).split(",");
++ String paths[] = new String(data, UTF_8).split(",");
+
+ Path orig = new Path(paths[0]);
+ Path dest = new Path(paths[1]);
+ Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
+
+ try {
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
+ FileSystem origFs = TraceFileSystem.wrap(vm.getVolumeByPath(orig).getFileSystem());
+ FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
+
+ FileUtil.copy(origFs, orig, destFs, tmp, false, true, CachedConfiguration.getInstance());
+ destFs.rename(tmp, dest);
+ log.debug("copied " + orig + " to " + dest);
+ } catch (IOException ex) {
+ try {
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
+ FileSystem destFs = TraceFileSystem.wrap(vm.getVolumeByPath(dest).getFileSystem());
+ destFs.create(dest).close();
+ log.warn(" marked " + dest + " failed", ex);
+ } catch (IOException e) {
+ log.error("Unable to create failure flag file " + dest, e);
+ }
+ }
+
+ }
+
+}
[23/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a338d5f,0000000..d5c940d
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@@ -1,553 -1,0 +1,553 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.DiskUsage;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class VolumeIT extends ConfigurableMacIT {
+
+ private static final Text EMPTY = new Text();
+ private static final Value EMPTY_VALUE = new Value(new byte[] {});
+ private File volDirBase;
+ private Path v1, v2;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ File baseDir = cfg.getDir();
+ volDirBase = new File(baseDir, "volumes");
+ File v1f = new File(volDirBase, "v1");
+ File v2f = new File(volDirBase, "v2");
+ v1f.mkdir();
+ v2f.mkdir();
+ v1 = new Path("file://" + v1f.getAbsolutePath());
+ v2 = new Path("file://" + v2f.getAbsolutePath());
+
+ // Run MAC on two locations in the local file system
+ URI v1Uri = v1.toUri();
+ cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
+ cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
+ cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+
+ super.configure(cfg, hadoopCoreSite);
+ }
+
+ @Test
+ public void test() throws Exception {
+ // create a table
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName);
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ // with some splits
+ for (String s : "d,m,t".split(","))
+ partitions.add(new Text(s));
+ connector.tableOperations().addSplits(tableName, partitions);
+ // scribble over the splits
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+ String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ for (String s : rows) {
+ Mutation m = new Mutation(new Text(s));
+ m.put(EMPTY, EMPTY, EMPTY_VALUE);
+ bw.addMutation(m);
+ }
+ bw.close();
+ // write the data to disk, read it back
+ connector.tableOperations().flush(tableName, null, null, true);
+ Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
+ int i = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ assertEquals(rows[i++], entry.getKey().getRow().toString());
+ }
+ // verify the new files are written to the different volumes
+ scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(new Range("1", "1<"));
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ int fileCount = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ boolean inV1 = entry.getKey().getColumnQualifier().toString().contains(v1.toString());
+ boolean inV2 = entry.getKey().getColumnQualifier().toString().contains(v2.toString());
+ assertTrue(inV1 || inV2);
+ fileCount++;
+ }
+ assertEquals(4, fileCount);
+ List<DiskUsage> diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
+ assertEquals(1, diskUsage.size());
+ long usage = diskUsage.get(0).getUsage().longValue();
+ System.out.println("usage " + usage);
+ assertTrue(usage > 700 && usage < 800);
+ }
+
+ private void verifyData(List<String> expected, Scanner createScanner) {
+
+ List<String> actual = new ArrayList<String>();
+
+ for (Entry<Key,Value> entry : createScanner) {
+ Key k = entry.getKey();
+ actual.add(k.getRow() + ":" + k.getColumnFamily() + ":" + k.getColumnQualifier() + ":" + entry.getValue());
+ }
+
+ Collections.sort(expected);
+ Collections.sort(actual);
+
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testRelativePaths() throws Exception {
+
+ List<String> expected = new ArrayList<String>();
+
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName, false);
+
+ String tableId = connector.tableOperations().tableIdMap().get(tableName);
+
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ // with some splits
+ for (String s : "c,g,k,p,s,v".split(","))
+ partitions.add(new Text(s));
+
+ connector.tableOperations().addSplits(tableName, partitions);
+
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+
+ // create two files in each tablet
+
+ String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ for (String s : rows) {
+ Mutation m = new Mutation(s);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ expected.add(s + ":cf1:cq1:1");
+ }
+
+ bw.flush();
+ connector.tableOperations().flush(tableName, null, null, true);
+
+ for (String s : rows) {
+ Mutation m = new Mutation(s);
+ m.put("cf1", "cq1", "2");
+ bw.addMutation(m);
+ expected.add(s + ":cf1:cq1:2");
+ }
+
+ bw.close();
+ connector.tableOperations().flush(tableName, null, null, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ connector.tableOperations().offline(tableName, true);
+
+ connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+
+ Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ String cq = entry.getKey().getColumnQualifier().toString();
+ if (cq.startsWith(v1.toString())) {
+ Path path = new Path(cq);
+ String relPath = "/" + path.getParent().getName() + "/" + path.getName();
+ Mutation fileMut = new Mutation(entry.getKey().getRow());
+ fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
+ mbw.addMutation(fileMut);
+ }
+ }
+
+ mbw.close();
+
+ connector.tableOperations().online(tableName, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ connector.tableOperations().compact(tableName, null, null, true, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ String cq = entry.getKey().getColumnQualifier().toString();
+ Path path = new Path(cq);
+ Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
+ }
+
+ }
+
+ @Test
+ public void testAddVolumes() throws Exception {
+
+ String[] tableNames = getUniqueNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v1.toString() + "," + v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ verifyVolumesUsed(tableNames[1], false, v1, v2, v3);
+ }
+
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
+
+ String[] tableNames = getUniqueNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
+
+ // v1 should not have any data for tableNames[1]
+ verifyVolumesUsed(tableNames[1], false, v2, v3);
+ }
+
+ private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
+ MutationsRejectedException {
+ TreeSet<Text> splits = new TreeSet<Text>();
+ for (int i = 1; i < 100; i++) {
+ splits.add(new Text(String.format("%06d", i * 100)));
+ }
+
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().addSplits(tableName, splits);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ Mutation m = new Mutation(row);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+
+ private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, MutationsRejectedException {
+
+ Connector conn = getConnector();
+
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ if (!conn.tableOperations().exists(tableName)) {
+ Assert.assertFalse(shouldExist);
+
+ writeData(tableName, conn);
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ conn.tableOperations().flush(tableName, null, null, true);
+ }
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
+ metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ int counts[] = new int[paths.length];
+
+ outer: for (Entry<Key,Value> entry : metaScanner) {
+ String cf = entry.getKey().getColumnFamily().toString();
+ String cq = entry.getKey().getColumnQualifier().toString();
+
+ String path;
+ if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
+ path = cq;
+ else
+ path = entry.getValue().toString();
+
+ for (int i = 0; i < paths.length; i++) {
+ if (path.startsWith(paths[i].toString())) {
+ counts[i]++;
+ continue outer;
+ }
+ }
+
+ Assert.fail("Unexpected volume " + path);
+ }
+
+ // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
+ // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
+
+ int sum = 0;
+ for (int count : counts) {
+ Assert.assertTrue(count > 0);
+ sum += count;
+ }
+
+ Assert.assertEquals(200, sum);
+ }
+
+ @Test
+ public void testRemoveVolumes() throws Exception {
+ String[] tableNames = getUniqueNames(2);
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volume was decommisioned
+ cluster.start();
+
+ Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
+ conn.tableOperations().compact(tableNames[0], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], true, v2);
+
+ // check that root tablet is not on volume 1
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
+ ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
- String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++ String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
+ Assert.assertTrue(rootTabletDir.startsWith(v2.toString()));
+ zookeeper.close();
+
+ conn.tableOperations().clone(tableNames[0], tableNames[1], true, new HashMap<String,String>(), new HashSet<String>());
+
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ conn.tableOperations().flush(RootTable.NAME, null, null, true);
+
+ verifyVolumesUsed(tableNames[0], true, v2);
+ verifyVolumesUsed(tableNames[1], true, v2);
+
+ }
+
+ private void testReplaceVolume(boolean cleanShutdown) throws Exception {
+ String[] tableNames = getUniqueNames(3);
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ // write to 2nd table, but do not flush data to disk before shutdown
+ writeData(tableNames[1], cluster.getConnector("root", ROOT_PASSWORD));
+
+ if (cleanShutdown)
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+
+ cluster.stop();
+
+ File v1f = new File(v1.toUri());
+ File v8f = new File(new File(v1.getParent().toUri()), "v8");
+ Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
+ Path v8 = new Path(v8f.toURI());
+
+ File v2f = new File(v2.toUri());
+ File v9f = new File(new File(v2.getParent().toUri()), "v9");
+ Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
+ Path v9 = new Path(v9f.toURI());
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
+ conf.set(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(), v1 + " " + v8 + "," + v2 + " " + v9);
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volumes were replaced
+ cluster.start();
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+
+ // verify writes to new dir
+ getConnector().tableOperations().compact(tableNames[0], null, null, true, true);
+ getConnector().tableOperations().compact(tableNames[1], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+
+ // check that root tablet is not on volume 1 or 2
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
+ ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
- String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++ String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
+ Assert.assertTrue(rootTabletDir.startsWith(v8.toString()) || rootTabletDir.startsWith(v9.toString()));
+ zookeeper.close();
+
+ getConnector().tableOperations().clone(tableNames[1], tableNames[2], true, new HashMap<String,String>(), new HashSet<String>());
+
+ getConnector().tableOperations().flush(MetadataTable.NAME, null, null, true);
+ getConnector().tableOperations().flush(RootTable.NAME, null, null, true);
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+ verifyVolumesUsed(tableNames[2], true, v8, v9);
+ }
+
+ @Test
+ public void testCleanReplaceVolumes() throws Exception {
+ testReplaceVolume(true);
+ }
+
+ @Test
+ public void testDirtyReplaceVolumes() throws Exception {
+ testReplaceVolume(false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
index f657034,0000000..05de342
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
@@@ -1,140 -1,0 +1,141 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class AddSplitIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void addSplitTest() throws Exception {
+
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+
+ insertData(tableName, 1l);
+
+ TreeSet<Text> splits = new TreeSet<Text>();
+ splits.add(new Text(String.format("%09d", 333)));
+ splits.add(new Text(String.format("%09d", 666)));
+
+ c.tableOperations().addSplits(tableName, splits);
+
+ UtilWaitThread.sleep(100);
+
+ Collection<Text> actualSplits = c.tableOperations().listSplits(tableName);
+
+ if (!splits.equals(new TreeSet<Text>(actualSplits))) {
+ throw new Exception(splits + " != " + actualSplits);
+ }
+
+ verifyData(tableName, 1l);
+ insertData(tableName, 2l);
+
+ // did not clear splits on purpose, it should ignore existing split points
+ // and still create the three additional split points
+
+ splits.add(new Text(String.format("%09d", 200)));
+ splits.add(new Text(String.format("%09d", 500)));
+ splits.add(new Text(String.format("%09d", 800)));
+
+ c.tableOperations().addSplits(tableName, splits);
+
+ UtilWaitThread.sleep(100);
+
+ actualSplits = c.tableOperations().listSplits(tableName);
+
+ if (!splits.equals(new TreeSet<Text>(actualSplits))) {
+ throw new Exception(splits + " != " + actualSplits);
+ }
+
+ verifyData(tableName, 2l);
+ }
+
+ private void verifyData(String tableName, long ts) throws Exception {
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ for (int i = 0; i < 10000; i++) {
+ if (!iter.hasNext()) {
+ throw new Exception("row " + i + " not found");
+ }
+
+ Entry<Key,Value> entry = iter.next();
+
+ String row = String.format("%09d", i);
+
+ if (!entry.getKey().getRow().equals(new Text(row))) {
+ throw new Exception("unexpected row " + entry.getKey() + " " + i);
+ }
+
+ if (entry.getKey().getTimestamp() != ts) {
+ throw new Exception("unexpected ts " + entry.getKey() + " " + ts);
+ }
+
+ if (Integer.parseInt(entry.getValue().toString()) != i) {
+ throw new Exception("unexpected value " + entry + " " + i);
+ }
+ }
+
+ if (iter.hasNext()) {
+ throw new Exception("found more than expected " + iter.next());
+ }
+
+ }
+
+ private void insertData(String tableName, long ts) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
+ BatchWriter bw = getConnector().createBatchWriter(tableName, null);
+
+ for (int i = 0; i < 10000; i++) {
+ String row = String.format("%09d", i);
+
+ Mutation m = new Mutation(new Text(row));
- m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
index 0139a10,0000000..9c4492e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
@@@ -1,110 -1,0 +1,110 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.EnumSet;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BadIteratorMincIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ IteratorSetting is = new IteratorSetting(30, BadIterator.class);
+ c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+
+ Mutation m = new Mutation(new Text("r1"));
- m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++ m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ bw.close();
+
+ c.tableOperations().flush(tableName, null, null, false);
+ UtilWaitThread.sleep(1000);
+
+ // minc should fail, so there should be no files
+ FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 0, 0);
+
+ // try to scan table
+ Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+ int count = FunctionalTestUtils.count(scanner);
+ assertEquals("Did not see expected # entries " + count, 1, count);
+
+ // remove the bad iterator
+ c.tableOperations().removeIterator(tableName, BadIterator.class.getSimpleName(), EnumSet.of(IteratorScope.minc));
+
+ UtilWaitThread.sleep(5000);
+
+ // minc should complete
+ FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 1, 1);
+
+ count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 1)
+ throw new Exception("Did not see expected # entries " + count);
+
+ // now try putting bad iterator back and deleting the table
+ c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
+ bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ m = new Mutation(new Text("r2"));
- m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++ m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
+ bw.addMutation(m);
+ bw.close();
+
+ // make sure property is given time to propagate
+ UtilWaitThread.sleep(500);
+
+ c.tableOperations().flush(tableName, null, null, false);
+
+ // make sure the flush has time to start
+ UtilWaitThread.sleep(1000);
+
+ // this should not hang
+ c.tableOperations().delete(tableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index c67407d,0000000..688a326
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@@ -1,125 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Random;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BatchScanSplitIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "0"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ int numRows = 1 << 18;
+
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 0; i < numRows; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ getConnector().tableOperations().flush(tableName, null, null, true);
+
+ getConnector().tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "4K");
+
+ Collection<Text> splits = getConnector().tableOperations().listSplits(tableName);
+ while (splits.size() < 2) {
+ UtilWaitThread.sleep(1);
+ splits = getConnector().tableOperations().listSplits(tableName);
+ }
+
+ System.out.println("splits : " + splits);
+
+ Random random = new Random(19011230);
+ HashMap<Text,Value> expected = new HashMap<Text,Value>();
+ ArrayList<Range> ranges = new ArrayList<Range>();
+ for (int i = 0; i < 100; i++) {
+ int r = random.nextInt(numRows);
+ Text row = new Text(String.format("%09x", r));
- expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(Constants.UTF8)));
++ expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(UTF_8)));
+ ranges.add(new Range(row));
+ }
+
+ // logger.setLevel(Level.TRACE);
+
+ HashMap<Text,Value> found = new HashMap<Text,Value>();
+
+ for (int i = 0; i < 20; i++) {
+ BatchScanner bs = getConnector().createBatchScanner(tableName, Authorizations.EMPTY, 4);
+
+ found.clear();
+
+ long t1 = System.currentTimeMillis();
+
+ bs.setRanges(ranges);
+
+ for (Entry<Key,Value> entry : bs) {
+ found.put(entry.getKey().getRow(), entry.getValue());
+ }
+ bs.close();
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("rate : %06.2f%n", ranges.size() / ((t2 - t1) / 1000.0)));
+
+ if (!found.equals(expected))
+ throw new Exception("Found and expected differ " + found + " " + expected);
+ }
+
+ splits = getConnector().tableOperations().listSplits(tableName);
+ log.info("splits : " + splits);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
index e2dd903,0000000..465936e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
@@@ -1,181 -1,0 +1,182 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BatchWriterFlushIT extends SimpleMacIT {
+
+ private static final int NUM_TO_FLUSH = 100000;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 90;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = getUniqueNames(2);
+ String bwft = tableNames[0];
+ c.tableOperations().create(bwft);
+ String bwlt = tableNames[1];
+ c.tableOperations().create(bwlt);
+ runFlushTest(bwft);
+ runLatencyTest(bwlt);
+
+ }
+
+ private void runLatencyTest(String tableName) throws Exception {
+ // should automatically flush after 2 seconds
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(1000, TimeUnit.MILLISECONDS));
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+
+ Mutation m = new Mutation(new Text(String.format("r_%10d", 1)));
- m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(Constants.UTF8)));
++ m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(UTF_8)));
+ bw.addMutation(m);
+
+ UtilWaitThread.sleep(500);
+
+ int count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 0) {
+ throw new Exception("Flushed too soon");
+ }
+
+ UtilWaitThread.sleep(1500);
+
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 1) {
+ throw new Exception("Did not flush");
+ }
+
+ bw.close();
+ }
+
+ private void runFlushTest(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException,
+ Exception {
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ Random r = new Random();
+
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < NUM_TO_FLUSH; j++) {
+ int row = i * NUM_TO_FLUSH + j;
+
+ Mutation m = new Mutation(new Text(String.format("r_%10d", row)));
+ m.put(new Text("cf"), new Text("cq"), new Value(("" + row).getBytes()));
+ bw.addMutation(m);
+ }
+
+ bw.flush();
+
+ // do a few random lookups into the data just flushed
+
+ for (int k = 0; k < 10; k++) {
+ int rowToLookup = r.nextInt(NUM_TO_FLUSH) + i * NUM_TO_FLUSH;
+
+ scanner.setRange(new Range(new Text(String.format("r_%10d", rowToLookup))));
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ if (!iter.hasNext())
+ throw new Exception(" row " + rowToLookup + " not found after flush");
+
+ Entry<Key,Value> entry = iter.next();
+
+ if (iter.hasNext())
+ throw new Exception("Scanner returned too much");
+
+ verifyEntry(rowToLookup, entry);
+ }
+
+ // scan all data just flushed
+ scanner.setRange(new Range(new Text(String.format("r_%10d", i * NUM_TO_FLUSH)), true, new Text(String.format("r_%10d", (i + 1) * NUM_TO_FLUSH)), false));
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ for (int j = 0; j < NUM_TO_FLUSH; j++) {
+ int row = i * NUM_TO_FLUSH + j;
+
+ if (!iter.hasNext())
+ throw new Exception("Scan stopped permaturely at " + row);
+
+ Entry<Key,Value> entry = iter.next();
+
+ verifyEntry(row, entry);
+ }
+
+ if (iter.hasNext())
+ throw new Exception("Scanner returned too much");
+
+ }
+
+ bw.close();
+
+ // test adding a mutation to a closed batch writer
+ boolean caught = false;
+ try {
+ bw.addMutation(new Mutation(new Text("foobar")));
+ } catch (IllegalStateException ise) {
+ caught = true;
+ }
+
+ if (!caught) {
+ throw new Exception("Adding to closed batch writer did not fail");
+ }
+ }
+
+ private void verifyEntry(int row, Entry<Key,Value> entry) throws Exception {
+ if (!entry.getKey().getRow().toString().equals(String.format("r_%10d", row))) {
+ throw new Exception("Unexpected key returned, expected " + row + " got " + entry.getKey());
+ }
+
+ if (!entry.getValue().toString().equals("" + row)) {
+ throw new Exception("Unexpected value, expected " + row + " got " + entry.getValue());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 7974c6c,0000000..ffe55bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@@ -1,120 -1,0 +1,121 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BulkFileIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void testBulkFile() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (String split : "0333 0666 0999 1333 1666".split(" "))
+ splits.add(new Text(split));
+ c.tableOperations().addSplits(tableName, splits);
+ Configuration conf = new Configuration();
+ AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
+
+ String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
+
+ fs.delete(new Path(dir), true);
+
+ FileSKVWriter writer1 = FileOperations.getInstance().openWriter(dir + "/f1." + RFile.EXTENSION, fs, conf, aconf);
+ writer1.startDefaultLocalityGroup();
+ writeData(writer1, 0, 333);
+ writer1.close();
+
+ FileSKVWriter writer2 = FileOperations.getInstance().openWriter(dir + "/f2." + RFile.EXTENSION, fs, conf, aconf);
+ writer2.startDefaultLocalityGroup();
+ writeData(writer2, 334, 999);
+ writer2.close();
+
+ FileSKVWriter writer3 = FileOperations.getInstance().openWriter(dir + "/f3." + RFile.EXTENSION, fs, conf, aconf);
+ writer3.startDefaultLocalityGroup();
+ writeData(writer3, 1000, 1999);
+ writer3.close();
+
+ FunctionalTestUtils.bulkImport(c, fs, tableName, dir);
+
+ FunctionalTestUtils.checkRFiles(c, tableName, 6, 6, 1, 1);
+
+ verifyData(tableName, 0, 1999);
+
+ }
+
+ private void verifyData(String table, int s, int e) throws Exception {
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ for (int i = s; i <= e; i++) {
+ if (!iter.hasNext())
+ throw new Exception("row " + i + " not found");
+
+ Entry<Key,Value> entry = iter.next();
+
+ String row = String.format("%04d", i);
+
+ if (!entry.getKey().getRow().equals(new Text(row)))
+ throw new Exception("unexpected row " + entry.getKey() + " " + i);
+
+ if (Integer.parseInt(entry.getValue().toString()) != i)
+ throw new Exception("unexpected value " + entry + " " + i);
+ }
+
+ if (iter.hasNext())
+ throw new Exception("found more than expected " + iter.next());
+ }
+
+ private void writeData(FileSKVWriter w, int s, int e) throws Exception {
+ for (int i = s; i <= e; i++) {
- w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(UTF_8)));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
index 785a997,0000000..92bd714
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
@@@ -1,147 -1,0 +1,148 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.EnumSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ConcurrencyIT extends ConfigurableMacIT {
+
+ static class ScanTask extends Thread {
+
+ int count = 0;
+ Scanner scanner;
+
+ ScanTask(Connector conn, long time) throws Exception {
+ scanner = conn.createScanner("cct", Authorizations.EMPTY);
+ IteratorSetting slow = new IteratorSetting(30, "slow", SlowIterator.class);
+ SlowIterator.setSleepTime(slow, time);
+ scanner.addScanIterator(slow);
+ }
+
+ @Override
+ public void run() {
+ count = FunctionalTestUtils.count(scanner);
+ }
+
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ /*
+ * Below is a diagram of the operations in this test over time.
+ *
+ * Scan 0 |------------------------------| Scan 1 |----------| Minc 1 |-----| Scan 2 |----------| Scan 3 |---------------| Minc 2 |-----| Majc 1 |-----|
+ */
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ runTest(c);
+ }
+
+ static void runTest(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
+ MutationsRejectedException, Exception, InterruptedException {
+ c.tableOperations().create("cct");
+ IteratorSetting is = new IteratorSetting(10, SlowIterator.class);
+ SlowIterator.setSleepTime(is, 50);
+ c.tableOperations().attachIterator("cct", is, EnumSet.of(IteratorScope.minc, IteratorScope.majc));
+ c.tableOperations().setProperty("cct", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
+
+ BatchWriter bw = c.createBatchWriter("cct", new BatchWriterConfig());
+ for (int i = 0; i < 50; i++) {
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+ bw.flush();
+
+ ScanTask st0 = new ScanTask(c, 300);
+ st0.start();
+
+ ScanTask st1 = new ScanTask(c, 100);
+ st1.start();
+
+ UtilWaitThread.sleep(50);
+ c.tableOperations().flush("cct", null, null, true);
+
+ for (int i = 0; i < 50; i++) {
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+
+ bw.flush();
+
+ ScanTask st2 = new ScanTask(c, 100);
+ st2.start();
+
+ st1.join();
+ st2.join();
+ if (st1.count != 50)
+ throw new Exception("Thread 1 did not see 50, saw " + st1.count);
+
+ if (st2.count != 50)
+ throw new Exception("Thread 2 did not see 50, saw " + st2.count);
+
+ ScanTask st3 = new ScanTask(c, 150);
+ st3.start();
+
+ UtilWaitThread.sleep(50);
+ c.tableOperations().flush("cct", null, null, false);
+
+ st3.join();
+ if (st3.count != 50)
+ throw new Exception("Thread 3 did not see 50, saw " + st3.count);
+
+ st0.join();
+ if (st0.count != 50)
+ throw new Exception("Thread 0 did not see 50, saw " + st0.count);
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
index d5e9188,0000000..7e5944e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@@ -1,317 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ConstraintIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 30;
+ }
+
+ @Test
+ public void run() throws Exception {
+ String[] tableNames = getUniqueNames(3);
+ Connector c = getConnector();
+ for (String table : tableNames) {
+ c.tableOperations().create(table);
+ c.tableOperations().addConstraint(table, NumericValueConstraint.class.getName());
+ c.tableOperations().addConstraint(table, AlphaNumKeyConstraint.class.getName());
+ }
+
+ // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
+ // logger.setLevel(Level.TRACE);
+
+ test1(tableNames[0]);
+
+ // logger.setLevel(Level.TRACE);
+
+ test2(tableNames[1], false);
+ test2(tableNames[2], true);
+ }
+
+ private void test1(String tableName) throws Exception {
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ Mutation mut1 = new Mutation(new Text("r1"));
- mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(Constants.UTF8)));
++ mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(UTF_8)));
+
+ bw.addMutation(mut1);
+
+ // should not throw any exceptions
+ bw.close();
+
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ // create a mutation with a non numeric value
+ Mutation mut2 = new Mutation(new Text("r1"));
- mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(Constants.UTF8)));
++ mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(UTF_8)));
+
+ bw.addMutation(mut2);
+
+ boolean sawMRE = false;
+
+ try {
+ bw.close();
+ // should not get here
+ throw new Exception("Test failed, constraint did not catch bad mutation");
+ } catch (MutationsRejectedException mre) {
+ sawMRE = true;
+
+ // verify constraint violation summary
+ List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
+
+ if (cvsl.size() != 1) {
+ throw new Exception("Unexpected constraints");
+ }
+
+ for (ConstraintViolationSummary cvs : cvsl) {
+ if (!cvs.constrainClass.equals(NumericValueConstraint.class.getName())) {
+ throw new Exception("Unexpected constraint class " + cvs.constrainClass);
+ }
+
+ if (cvs.numberOfViolatingMutations != 1) {
+ throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
+ }
+ }
+ }
+
+ if (!sawMRE) {
+ throw new Exception("Did not see MutationsRejectedException");
+ }
+
+ // verify mutation did not go through
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ scanner.setRange(new Range(new Text("r1")));
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ Entry<Key,Value> entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ // remove the numeric value constraint
+ getConnector().tableOperations().removeConstraint(tableName, 2);
+ UtilWaitThread.sleep(1000);
+
+ // now should be able to add a non numeric value
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+ bw.addMutation(mut2);
+ bw.close();
+
+ // verify mutation went through
+ iter = scanner.iterator();
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ // add a constraint that references a non-existant class
+ getConnector().tableOperations().setProperty(tableName, Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistantClass");
+ UtilWaitThread.sleep(1000);
+
+ // add a mutation
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ Mutation mut3 = new Mutation(new Text("r1"));
- mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++ mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
+
+ bw.addMutation(mut3);
+
+ sawMRE = false;
+
+ try {
+ bw.close();
+ // should not get here
+ throw new Exception("Test failed, mutation went through when table had bad constraints");
+ } catch (MutationsRejectedException mre) {
+ sawMRE = true;
+ }
+
+ if (!sawMRE) {
+ throw new Exception("Did not see MutationsRejectedException");
+ }
+
+ // verify the mutation did not go through
+ iter = scanner.iterator();
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ // remove the bad constraint
+ getConnector().tableOperations().removeConstraint(tableName, 1);
+ UtilWaitThread.sleep(1000);
+
+ // try the mutation again
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+ bw.addMutation(mut3);
+ bw.close();
+
+ // verify it went through
+ iter = scanner.iterator();
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+ }
+
+ private Mutation newMut(String row, String cf, String cq, String val) {
+ Mutation mut1 = new Mutation(new Text(row));
- mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(Constants.UTF8)));
++ mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(UTF_8)));
+ return mut1;
+ }
+
+ private void test2(String table, boolean doFlush) throws Exception {
+ // test sending multiple mutations with multiple constrain violations... all of the non violating mutations
+ // should go through
+ int numericErrors = 2;
+
+ BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ bw.addMutation(newMut("r1", "cf1", "cq1", "123"));
+ bw.addMutation(newMut("r1", "cf1", "cq2", "I'm a bad value"));
+ if (doFlush) {
+ try {
+ bw.flush();
+ throw new Exception("Didn't find a bad mutation");
+ } catch (MutationsRejectedException mre) {
+ // ignored
+ try {
+ bw.close();
+ } catch (MutationsRejectedException ex) {
+ // ignored
+ }
+ bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ numericErrors = 1;
+ }
+ }
+ bw.addMutation(newMut("r1", "cf1", "cq3", "I'm a naughty value"));
+ bw.addMutation(newMut("@bad row@", "cf1", "cq2", "456"));
+ bw.addMutation(newMut("r1", "cf1", "cq4", "789"));
+
+ boolean sawMRE = false;
+
+ try {
+ bw.close();
+ // should not get here
+ throw new Exception("Test failed, constraint did not catch bad mutation");
+ } catch (MutationsRejectedException mre) {
+ System.out.println(mre);
+
+ sawMRE = true;
+
+ // verify constraint violation summary
+ List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
+
+ if (cvsl.size() != 2) {
+ throw new Exception("Unexpected constraints");
+ }
+
+ HashMap<String,Integer> expected = new HashMap<String,Integer>();
+
+ expected.put("org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", numericErrors);
+ expected.put("org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint", 1);
+
+ for (ConstraintViolationSummary cvs : cvsl) {
+ if (expected.get(cvs.constrainClass) != cvs.numberOfViolatingMutations) {
+ throw new Exception("Unexpected " + cvs.constrainClass + " " + cvs.numberOfViolatingMutations);
+ }
+ }
+ }
+
+ if (!sawMRE) {
+ throw new Exception("Did not see MutationsRejectedException");
+ }
+
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ Entry<Key,Value> entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
index b9b54c7,0000000..5b5249b
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
@@@ -1,128 -1,0 +1,129 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class CreateAndUseIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private static SortedSet<Text> splits;
+
+ @BeforeClass
+ public static void createData() throws Exception {
+ splits = new TreeSet<Text>();
+
+ for (int i = 1; i < 256; i++) {
+ splits.add(new Text(String.format("%08x", i << 8)));
+ }
+ }
+
+ @Test
+ public void verifyDataIsPresent() throws Exception {
+ Text cf = new Text("cf1");
+ Text cq = new Text("cq1");
+
+ String tableName = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(tableName);
+ getConnector().tableOperations().addSplits(tableName, splits);
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 1; i < 257; i++) {
+ Mutation m = new Mutation(new Text(String.format("%08x", (i << 8) - 16)));
- m.put(cf, cq, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ m.put(cf, cq, new Value(Integer.toString(i).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ Scanner scanner1 = getConnector().createScanner(tableName, Authorizations.EMPTY);
+
+ int ei = 1;
+
+ for (Entry<Key,Value> entry : scanner1) {
+ Assert.assertEquals(String.format("%08x", (ei << 8) - 16), entry.getKey().getRow().toString());
+ Assert.assertEquals(Integer.toString(ei), entry.getValue().toString());
+
+ ei++;
+ }
+
+ Assert.assertEquals("Did not see expected number of rows", 257, ei);
+ }
+
+ @Test
+ public void createTableAndScan() throws Exception {
+ String table2 = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(table2);
+ getConnector().tableOperations().addSplits(table2, splits);
+ Scanner scanner2 = getConnector().createScanner(table2, Authorizations.EMPTY);
+ int count = 0;
+ for (Entry<Key,Value> entry : scanner2) {
+ if (entry != null)
+ count++;
+ }
+
+ if (count != 0) {
+ throw new Exception("Did not see expected number of entries, count = " + count);
+ }
+ }
+
+ @Test
+ public void createTableAndBatchScan() throws Exception {
+ ArrayList<Range> ranges = new ArrayList<Range>();
+ for (int i = 1; i < 257; i++) {
+ ranges.add(new Range(new Text(String.format("%08x", (i << 8) - 16))));
+ }
+
+ String table3 = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(table3);
+ getConnector().tableOperations().addSplits(table3, splits);
+ BatchScanner bs = getConnector().createBatchScanner(table3, Authorizations.EMPTY, 3);
+ bs.setRanges(ranges);
+ Iterator<Entry<Key,Value>> iter = bs.iterator();
+ int count = Iterators.size(iter);
+ bs.close();
+
+ Assert.assertEquals("Did not expect to find any entries", 0, count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
index 83b5b24,0000000..0578ef4
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
@@@ -1,94 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.*;
+
+import java.util.Collections;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class DeleteEverythingIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1s"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("de");
+ BatchWriter bw = getConnector().createBatchWriter("de", new BatchWriterConfig());
+ Mutation m = new Mutation(new Text("foo"));
- m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(Constants.UTF8)));
++ m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(UTF_8)));
+ bw.addMutation(m);
+ bw.flush();
+
+ getConnector().tableOperations().flush("de", null, null, true);
+
+ FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 1, 1);
+
+ m = new Mutation(new Text("foo"));
+ m.putDelete(new Text("bar"), new Text("1910"));
+ bw.addMutation(m);
+ bw.flush();
+
+ Scanner scanner = getConnector().createScanner("de", Authorizations.EMPTY);
+ scanner.setRange(new Range());
+ int count = FunctionalTestUtils.count(scanner);
+ assertEquals("count == " + count, 0, count);
+ getConnector().tableOperations().flush("de", null, null, true);
+
+ getConnector().tableOperations().setProperty("de", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
+ UtilWaitThread.sleep(4000);
+
+ FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 0, 0);
+
+ bw.close();
+
+ count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 0)
+ throw new Exception("count == " + count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
index a8237a6,0000000..d35ba9f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
@@@ -1,143 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+
+// attempt to reproduce ACCUMULO-315
+public class DeleteRowsSplitIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private static final Logger log = Logger.getLogger(DeleteRowsSplitIT.class);
+
+ private static final String LETTERS = "abcdefghijklmnopqrstuvwxyz";
+ static final SortedSet<Text> SPLITS = new TreeSet<Text>();
+ static final List<String> ROWS = new ArrayList<String>();
+ static {
- for (byte b : LETTERS.getBytes(Constants.UTF8)) {
++ for (byte b : LETTERS.getBytes(UTF_8)) {
+ SPLITS.add(new Text(new byte[] {b}));
- ROWS.add(new String(new byte[] {b}, Constants.UTF8));
++ ROWS.add(new String(new byte[] {b}, UTF_8));
+ }
+ }
+
+ @Test
+ public void run() throws Exception {
+ // Delete ranges of rows, and verify the are removed
+ // Do this while adding many splits
+ final String tableName = getUniqueNames(1)[0];
+
+ // Eliminate whole tablets
+ for (int test = 0; test < 10; test++) {
+ // create a table
+ log.info("Test " + test);
+ getConnector().tableOperations().create(tableName);
+
+ // put some data in it
+ fillTable(tableName);
+
+ // generate a random delete range
+ final Text start = new Text();
+ final Text end = new Text();
+ generateRandomRange(start, end);
+
+ // initiate the delete range
+ final boolean fail[] = {false};
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ // split the table
+ final SortedSet<Text> afterEnd = SPLITS.tailSet(new Text(end.toString() + "\0"));
+ getConnector().tableOperations().addSplits(tableName, afterEnd);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ synchronized (fail) {
+ fail[0] = true;
+ }
+ }
+ }
+ };
+ t.start();
+
+ UtilWaitThread.sleep(test * 2);
+
+ getConnector().tableOperations().deleteRows(tableName, start, end);
+
+ t.join();
+ synchronized (fail) {
+ assertTrue(!fail[0]);
+ }
+
+ // scan the table
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ assertTrue(row.compareTo(start) <= 0 || row.compareTo(end) > 0);
+ }
+
+ // delete the table
+ getConnector().tableOperations().delete(tableName);
+ }
+ }
+
+ private void generateRandomRange(Text start, Text end) {
+ List<String> bunch = new ArrayList<String>(ROWS);
+ Collections.shuffle(bunch);
+ if (bunch.get(0).compareTo((bunch.get(1))) < 0) {
+ start.set(bunch.get(0));
+ end.set(bunch.get(1));
+ } else {
+ start.set(bunch.get(1));
+ end.set(bunch.get(0));
+ }
+
+ }
+
+ private void fillTable(String table) throws Exception {
+ BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ for (String row : ROWS) {
+ Mutation m = new Mutation(row);
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+}
[17/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
index cd26743,0000000..8022c14
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
@@@ -1,210 -1,0 +1,211 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.nio.ByteBuffer;
- import java.nio.charset.Charset;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
+
+public class ValueTest {
+ private static final byte[] toBytes(String s) {
- return s.getBytes(Charset.forName("UTF-8"));
++ return s.getBytes(UTF_8);
+ }
+
+ private static final byte[] DATA = toBytes("data");
+ private static final ByteBuffer DATABUFF = ByteBuffer.allocate(DATA.length);
+ static {
+ DATABUFF.put(DATA);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ DATABUFF.rewind();
+ }
+
+ @Test
+ public void testDefault() {
+ Value v = new Value();
+ assertEquals(0, v.get().length);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullBytesConstructor() {
+ new Value((byte[]) null);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullCopyConstructor() {
+ new Value((Value) null);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullByteBufferConstructor() {
+ new Value((ByteBuffer)null);
+ }
+
+ @Test(expected=NullPointerException.class)
+ public void testNullSet() {
+ Value v = new Value();
+ v.set(null);
+ }
+
+ @Test
+ public void testByteArray() {
+ Value v = new Value(DATA);
+ assertArrayEquals(DATA, v.get());
+ assertSame(DATA, v.get());
+ }
+
+ @Test
+ public void testByteArrayCopy() {
+ Value v = new Value(DATA, true);
+ assertArrayEquals(DATA, v.get());
+ assertNotSame(DATA, v.get());
+ }
+
+ @Test
+ public void testByteBuffer() {
+ Value v = new Value(DATABUFF);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testByteBufferCopy() {
+ @SuppressWarnings("deprecation")
+ Value v = new Value(DATABUFF, true);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testValueCopy() {
+ Value ov = createMock(Value.class);
+ expect(ov.get()).andReturn(DATA);
+ expect(ov.getSize()).andReturn(4);
+ replay(ov);
+ Value v = new Value(ov);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testByteArrayOffsetLength() {
+ Value v = new Value(DATA, 0, 4);
+ assertArrayEquals(DATA, v.get());
+ }
+
+ @Test
+ public void testSet() {
+ Value v = new Value();
+ v.set(DATA);
+ assertArrayEquals(DATA, v.get());
+ assertSame(DATA, v.get());
+ }
+
+ @Test
+ public void testCopy() {
+ Value v = new Value();
+ v.copy(DATA);
+ assertArrayEquals(DATA, v.get());
+ assertNotSame(DATA, v.get());
+ }
+
+ @Test
+ public void testGetSize() {
+ Value v = new Value(DATA);
+ assertEquals(DATA.length, v.getSize());
+ }
+
+ @Test
+ public void testGetSizeDefault() {
+ Value v = new Value();
+ assertEquals(0, v.getSize());
+ }
+
+ @Test
+ public void testWriteRead() throws Exception {
+ Value v = new Value(DATA);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ v.write(dos);
+ dos.close();
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ Value v2 = new Value();
+ v2.readFields(dis);
+ dis.close();
+ assertArrayEquals(DATA, v2.get());
+ }
+
+ @Test
+ public void testHashCode() {
+ Value v1 = new Value(DATA);
+ Value v2 = new Value(DATA);
+ assertEquals(v1.hashCode(), v2.hashCode());
+ }
+
+ @Test
+ public void testCompareTo() {
+ Value v1 = new Value(DATA);
+ Value v2 = new Value(toBytes("datb"));
+ assertTrue(v1.compareTo(v2) < 0);
+ assertTrue(v2.compareTo(v1) > 0);
+ Value v1a = new Value(DATA);
+ assertTrue(v1.compareTo(v1a) == 0);
+ Value v3 = new Value(toBytes("datc"));
+ assertTrue(v2.compareTo(v3) < 0);
+ assertTrue(v1.compareTo(v3) < 0);
+ }
+
+ @Test
+ public void testEquals() {
+ Value v1 = new Value(DATA);
+ assertTrue(v1.equals(v1));
+ Value v2 = new Value(DATA);
+ assertTrue(v1.equals(v2));
+ assertTrue(v2.equals(v1));
+ Value v3 = new Value(toBytes("datb"));
+ assertFalse(v1.equals(v3));
+ }
+
+ @Test
+ public void testToArray() {
+ List<byte[]> l = new java.util.ArrayList<byte[]>();
+ byte[] one = toBytes("one");
+ byte[] two = toBytes("two");
+ byte[] three = toBytes("three");
+ l.add(one);
+ l.add(two);
+ l.add(three);
+
+ byte[][] a = Value.toArray(l);
+ assertEquals(3, a.length);
+ assertArrayEquals(one, a[0]);
+ assertArrayEquals(two, a[1]);
+ assertArrayEquals(three, a[2]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index 931ff41,6c4e814..2577e48
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@@ -16,19 -16,11 +16,19 @@@
*/
package org.apache.accumulo.core.security;
++import static com.google.common.base.Charsets.UTF_8;
import static org.apache.accumulo.core.security.ColumnVisibility.quote;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.util.Comparator;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.security.ColumnVisibility.Node;
+import org.apache.accumulo.core.security.ColumnVisibility.NodeComparator;
+import org.apache.accumulo.core.security.ColumnVisibility.NodeType;
+import org.apache.hadoop.io.Text;
import org.junit.Test;
public class ColumnVisibilityTest {
@@@ -166,83 -152,4 +166,83 @@@
cv = new ColumnVisibility(quote("五"));
assertEquals("[\"五\"]", cv.toString());
}
+
+ @Test
+ public void testParseTree() {
+ Node node = parse("(W)|(U&V)");
+ assertNode(node, NodeType.OR, 0, 9);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
+ assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
+ }
+
+ @Test
+ public void testParseTreeWithNoChildren() {
+ Node node = parse("ABC");
+ assertNode(node, NodeType.TERM, 0, 3);
+ }
+
+ @Test
+ public void testParseTreeWithTwoChildren() {
+ Node node = parse("ABC|DEF");
+ assertNode(node, NodeType.OR, 0, 7);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
+ assertNode(node.getChildren().get(1), NodeType.TERM, 4, 7);
+ }
+
+ @Test
+ public void testParseTreeWithParenthesesAndTwoChildren() {
+ Node node = parse("(ABC|DEF)");
+ assertNode(node, NodeType.OR, 1, 8);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 1, 4);
+ assertNode(node.getChildren().get(1), NodeType.TERM, 5, 8);
+ }
+
+ @Test
+ public void testParseTreeWithParenthesizedChildren() {
+ Node node = parse("ABC|(DEF&GHI)");
+ assertNode(node, NodeType.OR, 0, 13);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 0, 3);
+ assertNode(node.getChildren().get(1), NodeType.AND, 5, 12);
+ assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 8);
+ assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 9, 12);
+ }
+
+ @Test
+ public void testParseTreeWithMoreParentheses() {
+ Node node = parse("(W)|(U&V)");
+ assertNode(node, NodeType.OR, 0, 9);
+ assertNode(node.getChildren().get(0), NodeType.TERM, 1, 2);
+ assertNode(node.getChildren().get(1), NodeType.AND, 5, 8);
+ assertNode(node.getChildren().get(1).children.get(0), NodeType.TERM, 5, 6);
+ assertNode(node.getChildren().get(1).children.get(1), NodeType.TERM, 7, 8);
+ }
+
+ @Test
+ public void testEmptyParseTreesAreEqual() {
+ Comparator<Node> comparator = new NodeComparator(new byte[] {});
+ Node empty = new ColumnVisibility().getParseTree();
+ assertEquals(0, comparator.compare(empty, parse("")));
+ }
+
+ @Test
+ public void testParseTreesOrdering() {
- byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(Constants.UTF8);
++ byte[] expression = "(b&c&d)|((a|m)&y&z)|(e&f)".getBytes(UTF_8);
+ byte[] flattened = new ColumnVisibility(expression).flatten();
+
+ // Convert to String for indexOf convenience
- String flat = new String(flattened, Constants.UTF8);
++ String flat = new String(flattened, UTF_8);
+ assertTrue("shortest expressions sort first", flat.indexOf('e') < flat.indexOf('|'));
+ assertTrue("shortest children sort first", flat.indexOf('b') < flat.indexOf('a'));
+ }
+
+ private Node parse(String s) {
+ ColumnVisibility v = new ColumnVisibility(s);
+ return v.getParseTree();
+ }
+
+ private void assertNode(Node node, NodeType nodeType, int start, int end) {
+ assertEquals(node.type, nodeType);
+ assertEquals(start, node.start);
+ assertEquals(end, node.end);
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
index b925e4a,0000000..a4e9f02
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
@@@ -1,143 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import javax.security.auth.DestroyFailedException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.client.security.tokens.NullToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class CredentialsTest {
+
+ @Test
+ public void testToThrift() throws DestroyFailedException {
+ // verify thrift serialization
+ Credentials creds = new Credentials("test", new PasswordToken("testing"));
+ TCredentials tCreds = creds.toThrift(new MockInstance());
+ assertEquals("test", tCreds.getPrincipal());
+ assertEquals(PasswordToken.class.getName(), tCreds.getTokenClassName());
+ assertArrayEquals(AuthenticationTokenSerializer.serialize(new PasswordToken("testing")), tCreds.getToken());
+
+ // verify that we can't serialize if it's destroyed
+ creds.getToken().destroy();
+ try {
+ creds.toThrift(new MockInstance());
+ fail();
+ } catch (Exception e) {
+ assertTrue(e instanceof RuntimeException);
+ assertTrue(e.getCause() instanceof AccumuloSecurityException);
+ assertTrue(AccumuloSecurityException.class.cast(e.getCause()).getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
+ }
+ }
+
+ @Test
+ public void roundtripThrift() throws DestroyFailedException {
+ Credentials creds = new Credentials("test", new PasswordToken("testing"));
+ TCredentials tCreds = creds.toThrift(new MockInstance());
+ Credentials roundtrip = Credentials.fromThrift(tCreds);
+ assertEquals("Roundtrip through thirft changed credentials equality", creds, roundtrip);
+ }
+
+ @Test
+ public void testMockConnector() throws AccumuloException, DestroyFailedException, AccumuloSecurityException {
+ Instance inst = new MockInstance();
+ Connector rootConnector = inst.getConnector("root", new PasswordToken());
+ PasswordToken testToken = new PasswordToken("testPass");
+ rootConnector.securityOperations().createLocalUser("testUser", testToken);
+
+ assertFalse(testToken.isDestroyed());
+ testToken.destroy();
+ assertTrue(testToken.isDestroyed());
+ try {
+ inst.getConnector("testUser", testToken);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ assertTrue(e.getSecurityErrorCode().equals(SecurityErrorCode.TOKEN_EXPIRED));
+ }
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ Credentials nullNullCreds = new Credentials(null, null);
+ Credentials abcNullCreds = new Credentials("abc", new NullToken());
+ Credentials cbaNullCreds = new Credentials("cba", new NullToken());
+ Credentials abcBlahCreds = new Credentials("abc", new PasswordToken("blah"));
+
+ // check hash codes
+ assertEquals(0, nullNullCreds.hashCode());
+ assertEquals("abc".hashCode(), abcNullCreds.hashCode());
+ assertEquals(abcNullCreds.hashCode(), abcBlahCreds.hashCode());
+ assertFalse(abcNullCreds.hashCode() == cbaNullCreds.hashCode());
+
+ // identity
+ assertEquals(abcNullCreds, abcNullCreds);
+ assertEquals(new Credentials("abc", new NullToken()), abcNullCreds);
+ // equal, but different token constructors
- assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(Constants.UTF8))), new Credentials("abc", new PasswordToken("abc")));
++ assertEquals(new Credentials("abc", new PasswordToken("abc".getBytes(UTF_8))), new Credentials("abc", new PasswordToken("abc")));
+ // test not equals
+ assertFalse(nullNullCreds.equals(abcBlahCreds));
+ assertFalse(nullNullCreds.equals(abcNullCreds));
+ assertFalse(abcNullCreds.equals(abcBlahCreds));
+ }
+
+ @Test
+ public void testCredentialsSerialization() throws AccumuloSecurityException {
- Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(Constants.UTF8)));
++ Credentials creds = new Credentials("a:b-c", new PasswordToken("d-e-f".getBytes(UTF_8)));
+ String serialized = creds.serialize();
+ Credentials result = Credentials.deserialize(serialized);
+ assertEquals(creds, result);
+ assertEquals("a:b-c", result.getPrincipal());
+ assertEquals(new PasswordToken("d-e-f"), result.getToken());
+
+ Credentials nullNullCreds = new Credentials(null, null);
+ serialized = nullNullCreds.serialize();
+ result = Credentials.deserialize(serialized);
+ assertEquals(null, result.getPrincipal());
+ assertEquals(null, result.getToken());
+ }
+
+ @Test
+ public void testToString() {
+ Credentials creds = new Credentials(null, null);
+ assertEquals(Credentials.class.getName() + ":null:null:<hidden>", creds.toString());
+ creds = new Credentials("", new NullToken());
+ assertEquals(Credentials.class.getName() + "::" + NullToken.class.getName() + ":<hidden>", creds.toString());
+ creds = new Credentials("abc", null);
+ assertEquals(Credentials.class.getName() + ":abc:null:<hidden>", creds.toString());
+ creds = new Credentials("abc", new PasswordToken(""));
+ assertEquals(Credentials.class.getName() + ":abc:" + PasswordToken.class.getName() + ":<hidden>", creds.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index d31a788,0000000..523279a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -1,103 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.Mutation;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class VisibilityConstraintTest {
+
+ VisibilityConstraint vc;
+ Environment env;
+ Mutation mutation;
+
+ static final ColumnVisibility good = new ColumnVisibility("good");
+ static final ColumnVisibility bad = new ColumnVisibility("bad");
+
+ static final String D = "don't care";
+
+ static final List<Short> ENOAUTH = Arrays.asList((short) 2);
+
+ @Before
+ public void setUp() throws Exception {
+ vc = new VisibilityConstraint();
+ mutation = new Mutation("r");
+
- ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8));
++ ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(UTF_8));
+
+ AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class);
+ expect(ac.contains(bs)).andReturn(true);
+ replay(ac);
+
+ env = createMock(Environment.class);
+ expect(env.getAuthorizationsContainer()).andReturn(ac);
+ replay(env);
+ }
+
+ @Test
+ public void testNoVisibility() {
+ mutation.put(D, D, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testVisibilityNoAuth() {
+ mutation.put(D, D, bad, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ public void testGoodVisibilityAuth() {
+ mutation.put(D, D, good, D);
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testCachedVisibilities() {
+ mutation.put(D, D, good, "v");
+ mutation.put(D, D, good, "v2");
+ assertNull("authorized", vc.check(env, mutation));
+ }
+
+ @Test
+ public void testMixedVisibilities() {
+ mutation.put(D, D, bad, D);
+ mutation.put(D, D, good, D);
+ assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation));
+ }
+
+ @Test
+ @Ignore
+ public void testMalformedVisibility() {
+ // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index ea64ac7,0000000..9078ef9
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -1,162 -1,0 +1,162 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security.crypto;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+
- import org.apache.accumulo.core.Constants;
+import org.junit.Test;
+
+public class BlockedIOStreamTest {
+ @Test
+ public void testLargeBlockIO() throws IOException {
+ writeRead(1024, 2048);
+ }
+
+ private void writeRead(int blockSize, int expectedSize) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, 1);
+
+ String contentString = "My Blocked Content String";
- byte[] content = contentString.getBytes(Constants.UTF8);
++ byte[] content = contentString.getBytes(UTF_8);
+ blockOut.write(content);
+ blockOut.flush();
+
+ String contentString2 = "My Other Blocked Content String";
- byte[] content2 = contentString2.getBytes(Constants.UTF8);
++ byte[] content2 = contentString2.getBytes(UTF_8);
+ blockOut.write(content2);
+ blockOut.flush();
+
+ blockOut.close();
+ byte[] written = baos.toByteArray();
+ assertEquals(expectedSize, written.length);
+
+ ByteArrayInputStream biis = new ByteArrayInputStream(written);
+ BlockedInputStream blockIn = new BlockedInputStream(biis, blockSize, blockSize);
+ DataInputStream dIn = new DataInputStream(blockIn);
+
+ dIn.readFully(content, 0, content.length);
- String readContentString = new String(content, Constants.UTF8);
++ String readContentString = new String(content, UTF_8);
+
+ assertEquals(contentString, readContentString);
+
+ dIn.readFully(content2, 0, content2.length);
- String readContentString2 = new String(content2, Constants.UTF8);
++ String readContentString2 = new String(content2, UTF_8);
+
+ assertEquals(contentString2, readContentString2);
+
+ blockIn.close();
+ }
+
+ @Test
+ public void testSmallBufferBlockedIO() throws IOException {
+ writeRead(16, (12 + 4) * (int) (Math.ceil(25.0 / 12) + Math.ceil(31.0 / 12)));
+ }
+
+ @Test
+ public void testSpillingOverOutputStream() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ // buffer will be size 12
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, 16, 16);
+ Random r = new Random(22);
+
+ byte[] undersized = new byte[11];
+ byte[] perfectSized = new byte[12];
+ byte[] overSized = new byte[13];
+ byte[] perfectlyOversized = new byte[13];
+ byte filler = (byte) r.nextInt();
+
+ r.nextBytes(undersized);
+ r.nextBytes(perfectSized);
+ r.nextBytes(overSized);
+ r.nextBytes(perfectlyOversized);
+
+ // 1 block
+ blockOut.write(undersized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ // 2 blocks
+ blockOut.write(perfectSized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ // 2 blocks
+ blockOut.write(overSized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ // 3 blocks
+ blockOut.write(undersized);
+ blockOut.write(perfectlyOversized);
+ blockOut.write(filler);
+ blockOut.flush();
+
+ blockOut.close();
+ assertEquals(16 * 8, baos.toByteArray().length);
+ }
+
+ @Test
+ public void testGiantWrite() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int blockSize = 16;
+ // buffer will be size 12
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, blockSize);
+ Random r = new Random(22);
+
+ int size = 1024 * 1024 * 128;
+ byte[] giant = new byte[size];
+ byte[] pattern = new byte[1024];
+
+ r.nextBytes(pattern);
+
+ for (int i = 0; i < size / 1024; i++) {
+ System.arraycopy(pattern, 0, giant, i * 1024, 1024);
+ }
+
+ blockOut.write(giant);
+ blockOut.flush();
+
+ blockOut.close();
+ baos.close();
+
+ int blocks = (int) Math.ceil(size / (blockSize - 4.0));
+ byte[] byteStream = baos.toByteArray();
+
+ assertEquals(blocks * 16, byteStream.length);
+
+ DataInputStream blockIn = new DataInputStream(new BlockedInputStream(new ByteArrayInputStream(byteStream), blockSize, blockSize));
+ Arrays.fill(giant, (byte) 0);
+ blockIn.readFully(giant, 0, size);
+ blockIn.close();
+
+ for (int i = 0; i < size / 1024; i++) {
+ byte[] readChunk = new byte[1024];
+ System.arraycopy(giant, i * 1024, readChunk, 0, 1024);
+ assertArrayEquals(pattern, readChunk);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
index 36def05,0000000..a2e4f7a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/DeleterFormatterTest.java
@@@ -1,176 -1,0 +1,176 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.format;
+
- import static org.apache.accumulo.core.Constants.UTF8;
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+
+import jline.UnsupportedTerminal;
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DeleterFormatterTest {
+ DeleterFormatter formatter;
+ Map<Key,Value> data;
+ BatchWriter writer;
+ BatchWriter exceptionWriter;
+ Shell shellState;
+
+ ByteArrayOutputStream baos;
+ ConsoleReader reader;
+
+ SettableInputStream input;
+
+ class SettableInputStream extends InputStream {
+ ByteArrayInputStream bais;
+
+ @Override
+ public int read() throws IOException {
+ return bais.read();
+ }
+
+ public void set(String in) {
- bais = new ByteArrayInputStream(in.getBytes(UTF8));
++ bais = new ByteArrayInputStream(in.getBytes(UTF_8));
+ }
+ };
+
+ @Before
+ public void setUp() throws IOException, MutationsRejectedException {
+ input = new SettableInputStream();
+ baos = new ByteArrayOutputStream();
+
+ MutationsRejectedException mre = createMock(MutationsRejectedException.class);
+
+ writer = createNiceMock(BatchWriter.class);
+ exceptionWriter = createNiceMock(BatchWriter.class);
+ exceptionWriter.close();
+ expectLastCall().andThrow(mre);
+ exceptionWriter.addMutation(anyObject(Mutation.class));
+ expectLastCall().andThrow(mre);
+
+ shellState = createNiceMock(Shell.class);
+
+ reader = new ConsoleReader(input, baos, new UnsupportedTerminal());
+ expect(shellState.getReader()).andReturn(reader).anyTimes();
+
+ replay(writer, exceptionWriter, shellState);
+
+ data = new TreeMap<Key,Value>();
- data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF8)));
++ data.put(new Key("r", "cf", "cq"), new Value("value".getBytes(UTF_8)));
+ }
+
+ @Test
+ public void testEmpty() {
+ formatter = new DeleterFormatter(writer, Collections.<Key,Value> emptyMap().entrySet(), true, shellState, true);
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testSingle() throws IOException {
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, true);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[DELETED]", " r ", "cf", "cq", "value");
+ }
+
+ @Test
+ public void testNo() throws IOException {
+ input.set("no\n");
- data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[SKIPPED]", " r ", "cf", "cq", "value");
+
+ assertTrue(formatter.hasNext());
+ }
+
+ @Test
+ public void testNoConfirmation() throws IOException {
+ input.set("");
- data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+
+ verify("[SKIPPED]", " r ", "cf", "cq", "value");
+
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testYes() throws IOException {
+ input.set("y\nyes\n");
- data.put(new Key("z"), new Value("v2".getBytes(UTF8)));
++ data.put(new Key("z"), new Value("v2".getBytes(UTF_8)));
+ formatter = new DeleterFormatter(writer, data.entrySet(), true, shellState, false);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ verify("[DELETED]", " r ", "cf", "cq", "value");
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ verify("[DELETED]", " z ", "v2");
+ }
+
+ @Test
+ public void testMutationException() {
+ formatter = new DeleterFormatter(exceptionWriter, data.entrySet(), true, shellState, true);
+
+ assertTrue(formatter.hasNext());
+ assertNull(formatter.next());
+ assertFalse(formatter.hasNext());
+ }
+
+ private void verify(String... chunks) throws IOException {
+ reader.flush();
+
+ String output = baos.toString();
+ for (String chunk : chunks) {
+ assertTrue(output.contains(chunk));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 8fc68fd,0000000..eef06c3
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@@ -1,90 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.format;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ShardedTableDistributionFormatterTest {
+ ShardedTableDistributionFormatter formatter;
+
+ Map<Key,Value> data;
+
+ @Before
+ public void setUp() {
+ data = new TreeMap<Key,Value>();
+ formatter = new ShardedTableDistributionFormatter();
+ }
+
+ @Test
+ public void testInitialize() {
+ data.put(new Key(), new Value());
+ data.put(new Key("r", "~tab"), new Value());
+ formatter.initialize(data.entrySet(), false);
+
+ assertTrue(formatter.hasNext());
+ formatter.next();
+ assertFalse(formatter.hasNext());
+ }
+
+ @Test
+ public void testAggregate() {
- data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(Constants.UTF8)));
- data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(Constants.UTF8)));
- data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(Constants.UTF8)));
++ data.put(new Key("t", "~tab", "loc"), new Value("srv1".getBytes(UTF_8)));
++ data.put(new Key("t;19700101", "~tab", "loc", 0), new Value("srv1".getBytes(UTF_8)));
++ data.put(new Key("t;19700101", "~tab", "loc", 1), new Value("srv2".getBytes(UTF_8)));
+
+ formatter.initialize(data.entrySet(), false);
+
+ String[] resultLines = formatter.next().split("\n");
+ List<String> results = Arrays.asList(resultLines).subList(2, 4);
+
+ assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("NULL", 1)));
+ assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("19700101", 2)));
+
+ assertFalse(formatter.hasNext());
+ }
+
+ private static class AggregateReportChecker implements Predicate {
+ private String day;
+ private int count;
+
+ AggregateReportChecker(String day, int count) {
+ this.day = day;
+ this.count = count;
+ }
+
+ @Override
+ public boolean evaluate(Object arg) {
+ String resLine = (String) arg;
+ return resLine.startsWith(this.day) && resLine.endsWith("" + this.count);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 1f8c32d,e411d22..af31cf2
--- a/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@@ -16,11 -16,10 +16,12 @@@
*/
package org.apache.accumulo.fate;
- import java.nio.charset.Charset;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@@ -37,34 -35,8 +38,33 @@@ import org.apache.zookeeper.KeeperExcep
* A utility to administer FATE operations
*/
public class AdminUtil<T> {
- private static final Charset UTF8 = Charset.forName("UTF-8");
+ private boolean exitOnError = false;
+
+ /**
+ * Default constructor
+ */
+ public AdminUtil() {
+ this(true);
+ }
+
+ /**
+ * Constructor
+ *
+ * @param exitOnError
+ * <code>System.exit(1)</code> on error if true
+ */
+ public AdminUtil(boolean exitOnError) {
+ super();
+ this.exitOnError = exitOnError;
+ }
+
public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath) throws KeeperException, InterruptedException {
+ print(zs, zk, lockPath, new Formatter(System.out), null, null);
+ }
+
+ public void print(ReadOnlyTStore<T> zs, IZooReaderWriter zk, String lockPath, Formatter fmt, Set<Long> filterTxid, EnumSet<TStatus> filterStatus)
+ throws KeeperException, InterruptedException {
Map<Long,List<String>> heldLocks = new HashMap<Long,List<String>>();
Map<Long,List<String>> waitingLocks = new HashMap<Long,List<String>>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index b6890c6,3ef6b0a..ff35364
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -32,8 -33,7 +33,7 @@@ import org.apache.zookeeper.ZooKeeper
import org.apache.zookeeper.ZooKeeper.States;
public class ZooSession {
- private static final Charset UTF8 = Charset.forName("UTF-8");
-
+
public static class ZooSessionShutdownException extends RuntimeException {
private static final long serialVersionUID = 1L;
@@@ -46,19 -46,18 +46,19 @@@
public ZooSessionInfo(ZooKeeper zooKeeper, ZooWatcher watcher) {
this.zooKeeper = zooKeeper;
}
-
+
ZooKeeper zooKeeper;
}
-
+
private static Map<String,ZooSessionInfo> sessions = new HashMap<String,ZooSessionInfo>();
-
+
private static String sessionKey(String keepers, int timeout, String scheme, byte[] auth) {
- return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF8));
+ return keepers + ":" + timeout + ":" + (scheme == null ? "" : scheme) + ":" + (auth == null ? "" : new String(auth, UTF_8));
}
-
+
private static class ZooWatcher implements Watcher {
-
+
+ @Override
public void process(WatchedEvent event) {
if (event.getState() == KeeperState.Expired) {
log.debug("Session expired, state of current session : " + event.getState());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index d5dc1e9,0000000..8216441
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -1,851 -1,0 +1,853 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.cluster.AccumuloCluster;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.gc.SimpleGarbageCollector;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.AccumuloStatus;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
+import org.apache.accumulo.start.Main;
+import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.commons.configuration.MapConfiguration;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Maps;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than {@link org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower.
+ *
+ * @since 1.6.0
+ */
+public class MiniAccumuloClusterImpl implements AccumuloCluster {
+ private static final Logger log = LoggerFactory.getLogger(MiniAccumuloClusterImpl.class);
+
+ public static class LogWriter extends Daemon {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {}
+ }
+ }
+
+ private boolean initialized = false;
+ private Process zooKeeperProcess = null;
+ private Process masterProcess = null;
+ private Process gcProcess = null;
+ private List<Process> tabletServerProcesses = Collections.synchronizedList(new ArrayList<Process>());
+
+ private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
+
+ private File zooCfgFile;
+ private String dfsUri;
+
+ public List<LogWriter> getLogWriters() {
+ return logWriters;
+ }
+
+ private List<LogWriter> logWriters = new ArrayList<MiniAccumuloClusterImpl.LogWriter>();
+
+ private MiniAccumuloConfigImpl config;
+ private MiniDFSCluster miniDFS = null;
+ private List<Process> cleanup = new ArrayList<Process>();
+
+ private ExecutorService executor;
+
+ public Process exec(Class<?> clazz, String... args) throws IOException {
+ return exec(clazz, null, args);
+ }
+
+ public Process exec(Class<?> clazz, List<String> jvmArgs, String... args) throws IOException {
+ ArrayList<String> jvmArgs2 = new ArrayList<String>(1 + (jvmArgs == null ? 0 : jvmArgs.size()));
+ jvmArgs2.add("-Xmx" + config.getDefaultMemory());
+ if (jvmArgs != null)
+ jvmArgs2.addAll(jvmArgs);
+ Process proc = _exec(clazz, jvmArgs2, args);
+ cleanup.add(proc);
+ return proc;
+ }
+
+ private boolean containsSiteFile(File f) {
+ return f.isDirectory() && f.listFiles(new FileFilter() {
+
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().endsWith("site.xml");
+ }
+ }).length > 0;
+ }
+
+ private void append(StringBuilder classpathBuilder, URL url) throws URISyntaxException {
+ File file = new File(url.toURI());
+ // do not include dirs containing hadoop or accumulo site files
+ if (!containsSiteFile(file))
+ classpathBuilder.append(File.pathSeparator).append(file.getAbsolutePath());
+ }
+
+ private String getClasspath() throws IOException {
+
+ try {
+ ArrayList<ClassLoader> classloaders = new ArrayList<ClassLoader>();
+
+ ClassLoader cl = this.getClass().getClassLoader();
+
+ while (cl != null) {
+ classloaders.add(cl);
+ cl = cl.getParent();
+ }
+
+ Collections.reverse(classloaders);
+
+ StringBuilder classpathBuilder = new StringBuilder();
+ classpathBuilder.append(config.getConfDir().getAbsolutePath());
+
+ if (config.getHadoopConfDir() != null)
+ classpathBuilder.append(File.pathSeparator).append(config.getHadoopConfDir().getAbsolutePath());
+
+ if (config.getClasspathItems() == null) {
+
+ // assume 0 is the system classloader and skip it
+ for (int i = 1; i < classloaders.size(); i++) {
+ ClassLoader classLoader = classloaders.get(i);
+
+ if (classLoader instanceof URLClassLoader) {
+
+ URLClassLoader ucl = (URLClassLoader) classLoader;
+
+ for (URL u : ucl.getURLs()) {
+ append(classpathBuilder, u);
+ }
+
+ } else if (classLoader instanceof VFSClassLoader) {
+
+ VFSClassLoader vcl = (VFSClassLoader) classLoader;
+ for (FileObject f : vcl.getFileObjects()) {
+ append(classpathBuilder, f.getURL());
+ }
+ } else {
+ throw new IllegalArgumentException("Unknown classloader type : " + classLoader.getClass().getName());
+ }
+ }
+ } else {
+ for (String s : config.getClasspathItems())
+ classpathBuilder.append(File.pathSeparator).append(s);
+ }
+
+ return classpathBuilder.toString();
+
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private Process _exec(Class<?> clazz, List<String> extraJvmOpts, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = getClasspath();
+
+ String className = clazz.getName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+ argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath));
+ argList.addAll(extraJvmOpts);
+ for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) {
+ argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue()));
+ }
+ argList.addAll(Arrays.asList("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-Dapple.awt.UIElement=true", Main.class.getName(),
+ className));
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath());
+ String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator);
+ builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath);
+ builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath);
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+ builder.environment().put("ACCUMULO_CONF_DIR", config.getConfDir().getAbsolutePath());
+ // hadoop-2.2 puts error messages in the logs if this is not set
+ builder.environment().put("HADOOP_HOME", config.getDir().getAbsolutePath());
+ if (config.getHadoopConfDir() != null)
+ builder.environment().put("HADOOP_CONF_DIR", config.getHadoopConfDir().getAbsolutePath());
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(config.getLogDir(), clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private Process _exec(Class<?> clazz, ServerType serverType, String... args) throws IOException {
+
+ List<String> jvmOpts = new ArrayList<String>();
+ jvmOpts.add("-Xmx" + config.getMemory(serverType));
+
+ if (config.isJDWPEnabled()) {
+ Integer port = PortUtils.getRandomFreePort();
+ jvmOpts.addAll(buildRemoteDebugParams(port));
+ debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
+ }
+ return _exec(clazz, jvmOpts, args);
+ }
+
+ /**
+ *
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ */
+ public MiniAccumuloClusterImpl(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfigImpl(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ */
+ @SuppressWarnings("deprecation")
+ public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException {
+
+ this.config = config.initialize();
+
+ config.getConfDir().mkdirs();
+ config.getLogDir().mkdirs();
+ config.getLibDir().mkdirs();
+ config.getLibExtDir().mkdirs();
+
+ if (!config.useExistingInstance()) {
+ config.getZooKeeperDir().mkdirs();
+ config.getWalogDir().mkdirs();
+ config.getAccumuloDir().mkdirs();
+ }
+
+ if (config.useMiniDFS()) {
+ File nn = new File(config.getAccumuloDir(), "nn");
+ nn.mkdirs();
+ File dn = new File(config.getAccumuloDir(), "dn");
+ dn.mkdirs();
+ File dfs = new File(config.getAccumuloDir(), "dfs");
+ dfs.mkdirs();
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
+ conf.set("dfs.support.append", "true");
+ conf.set("dfs.datanode.synconclose", "true");
+ conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
+ String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath());
+ miniDFS = new MiniDFSCluster(conf, 1, true, null);
+ if (oldTestBuildData == null)
+ System.clearProperty("test.build.data");
+ else
+ System.setProperty("test.build.data", oldTestBuildData);
+ miniDFS.waitClusterUp();
+ InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
+ dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
+ File coreFile = new File(config.getConfDir(), "core-site.xml");
+ writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
+ File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
+ writeConfig(hdfsFile, conf);
+
+ Map<String,String> siteConfig = config.getSiteConfig();
+ siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
+ siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
+ config.setSiteConfig(siteConfig);
+ } else if (config.useExistingInstance()) {
+ dfsUri = CachedConfiguration.getInstance().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
+ } else {
+ dfsUri = "file://";
+ }
+
+ File clientConfFile = config.getClientConfFile();
+ // Write only the properties that correspond to ClientConfiguration properties
+ writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate<Entry<String,String>>() {
+ @Override
+ public boolean apply(Entry<String,String> v) {
+ return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null;
+ }
+ }));
+
+ File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
+ writeConfig(siteFile, config.getSiteConfig().entrySet());
+
+ if (!config.useExistingInstance()) {
+ zooCfgFile = new File(config.getConfDir(), "zoo.cfg");
+ FileWriter fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "2000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", config.getZooKeeperPort() + "");
+ zooCfg.setProperty("maxClientCnxns", "1000");
+ zooCfg.setProperty("dataDir", config.getZooKeeperDir().getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+ }
+
+ // disable audit logging for mini....
+ InputStream auditStream = this.getClass().getResourceAsStream("/auditLog.xml");
+
+ if (auditStream != null) {
+ FileUtils.copyInputStreamToFile(auditStream, new File(config.getConfDir(), "auditLog.xml"));
+ }
+ }
+
+ private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings) throws IOException {
+ FileWriter fileWriter = new FileWriter(file);
+ fileWriter.append("<configuration>\n");
+
+ for (Entry<String,String> entry : settings) {
+ String value = entry.getValue().replace("&", "&").replace("<", "<").replace(">", ">");
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + value + "</value></property>\n");
+ }
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+ }
+
+ private void writeConfigProperties(File file, Map<String,String> settings) throws IOException {
+ FileWriter fileWriter = new FileWriter(file);
+
+ for (Entry<String,String> entry : settings.entrySet())
+ fileWriter.append(entry.getKey() + "=" + entry.getValue() + "\n");
+ fileWriter.close();
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IllegalStateException
+ * if already started
+ */
+ @Override
+ public synchronized void start() throws IOException, InterruptedException {
+ if (config.useExistingInstance()) {
+ Configuration acuConf = config.getAccumuloConfiguration();
+ Configuration hadoopConf = config.getHadoopConfiguration();
+
+ ConfigurationCopy cc = new ConfigurationCopy(acuConf);
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get(cc, hadoopConf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
+ IZooReaderWriter zrw = new ZooReaderWriterFactory().getZooReaderWriter(cc.get(Property.INSTANCE_ZK_HOST),
+ (int) cc.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), cc.get(Property.INSTANCE_SECRET));
+
+ String rootPath = ZooUtil.getRoot(instanceIdFromFile);
+
+ String instanceName = null;
+ try {
+ for (String name : zrw.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ byte[] bytes = zrw.getData(instanceNamePath, new Stat());
- String iid = new String(bytes, Constants.UTF8);
++ String iid = new String(bytes, UTF_8);
+ if (iid.equals(instanceIdFromFile)) {
+ instanceName = name;
+ }
+ }
+ } catch (KeeperException e) {
+ throw new RuntimeException("Unable to read instance name from zookeeper.", e);
+ }
+ if (instanceName == null)
+ throw new RuntimeException("Unable to read instance name from zookeeper.");
+
+ config.setInstanceName(instanceName);
+ if (!AccumuloStatus.isAccumuloOffline(zrw, rootPath))
+ throw new RuntimeException("The Accumulo instance being used is already running. Aborting.");
+ } else {
+ if (!initialized) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloClusterImpl.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ if (zooKeeperProcess == null) {
+ zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
+ }
+
+ if (!initialized) {
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ long startTime = System.currentTimeMillis();
+ while (true) {
+ Socket s = null;
+ try {
+ s = new Socket("localhost", config.getZooKeeperPort());
+ s.getOutputStream().write("ruok\n".getBytes());
+ s.getOutputStream().flush();
+ byte buffer[] = new byte[100];
+ int n = s.getInputStream().read(buffer);
+ if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
+ break;
+ } catch (Exception e) {
+ if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
+ throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in "
+ + config.getLogDir() + " for errors. Last exception: " + e);
+ }
+ } finally {
+ if (s != null)
+ s.close();
+ }
+ }
+ Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ initialized = true;
+ }
+ }
+
+ log.info("Starting MAC against instance {} and zookeeper(s) {}.", config.getInstanceName(), config.getZooKeepers());
+
+ synchronized (tabletServerProcesses) {
+ for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
+ tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
+ }
+ }
+ int ret = 0;
+ for (int i = 0; i < 5; i++) {
+ ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()).waitFor();
+ if (ret == 0)
+ break;
+ UtilWaitThread.sleep(1000);
+ }
+ if (ret != 0) {
+ throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ if (masterProcess == null) {
+ masterProcess = _exec(Master.class, ServerType.MASTER);
+ }
+
+ if (gcProcess == null) {
+ gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+ }
+
+ if (null == executor) {
+ executor = Executors.newSingleThreadExecutor();
+ }
+ }
+
+ private List<String> buildRemoteDebugParams(int port) {
+ return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d", port)});
+ }
+
+ /**
+ * @return generated remote debug ports if in debug mode.
+ * @since 1.6.0
+ */
+ public Set<Pair<ServerType,Integer>> getDebugPorts() {
+ return debugPorts;
+ }
+
+ List<ProcessReference> references(Process... procs) {
+ List<ProcessReference> result = new ArrayList<ProcessReference>();
+ for (Process proc : procs) {
+ result.add(new ProcessReference(proc));
+ }
+ return result;
+ }
+
+ public Map<ServerType,Collection<ProcessReference>> getProcesses() {
+ Map<ServerType,Collection<ProcessReference>> result = new HashMap<ServerType,Collection<ProcessReference>>();
+ result.put(ServerType.MASTER, references(masterProcess));
+ result.put(ServerType.TABLET_SERVER, references(tabletServerProcesses.toArray(new Process[0])));
+ result.put(ServerType.ZOOKEEPER, references(zooKeeperProcess));
+ if (null != gcProcess) {
+ result.put(ServerType.GARBAGE_COLLECTOR, references(gcProcess));
+ }
+ return result;
+ }
+
+ public void killProcess(ServerType type, ProcessReference proc) throws ProcessNotFoundException, InterruptedException {
+ boolean found = false;
+ switch (type) {
+ case MASTER:
+ if (proc.equals(masterProcess)) {
+ masterProcess.destroy();
+ masterProcess.waitFor();
+ masterProcess = null;
+ found = true;
+ }
+ break;
+ case TABLET_SERVER:
+ synchronized (tabletServerProcesses) {
+ for (Process tserver : tabletServerProcesses) {
+ if (proc.equals(tserver)) {
+ tabletServerProcesses.remove(tserver);
+ tserver.destroy();
+ tserver.waitFor();
+ found = true;
+ break;
+ }
+ }
+ }
+ break;
+ case ZOOKEEPER:
+ if (proc.equals(zooKeeperProcess)) {
+ zooKeeperProcess.destroy();
+ zooKeeperProcess.waitFor();
+ zooKeeperProcess = null;
+ found = true;
+ }
+ break;
+ case GARBAGE_COLLECTOR:
+ if (proc.equals(gcProcess)) {
+ gcProcess.destroy();
+ gcProcess.waitFor();
+ gcProcess = null;
+ found = true;
+ }
+ break;
+ }
+ if (!found)
+ throw new ProcessNotFoundException();
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+ @Override
+ public String getInstanceName() {
+ return config.getInstanceName();
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+ @Override
+ public String getZooKeepers() {
+ return config.getZooKeepers();
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. However its probably best to
+ * call stop in a finally block as soon as possible.
+ */
+ @Override
+ public synchronized void stop() throws IOException, InterruptedException {
+ if (null == executor) {
+ // keep repeated calls to stop() from failing
+ return;
+ }
+
+ for (LogWriter lw : logWriters) {
+ lw.flush();
+ }
+
+ if (gcProcess != null) {
+ try {
+ stopProcessWithTimeout(gcProcess, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("GarbageCollector did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("GarbageCollector did not fully stop after 30 seconds", e);
+ }
+ }
+ if (masterProcess != null) {
+ try {
+ stopProcessWithTimeout(masterProcess, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("Master did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("Master did not fully stop after 30 seconds", e);
+ }
+ }
+ if (tabletServerProcesses != null) {
+ synchronized (tabletServerProcesses) {
+ for (Process tserver : tabletServerProcesses) {
+ try {
+ stopProcessWithTimeout(tserver, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("TabletServer did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("TabletServer did not fully stop after 30 seconds", e);
+ }
+ }
+ }
+ }
+ if (zooKeeperProcess != null) {
+ try {
+ stopProcessWithTimeout(zooKeeperProcess, 30, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ log.warn("ZooKeeper did not fully stop after 30 seconds", e);
+ } catch (TimeoutException e) {
+ log.warn("ZooKeeper did not fully stop after 30 seconds", e);
+ }
+ }
+
+ zooKeeperProcess = null;
+ masterProcess = null;
+ gcProcess = null;
+ tabletServerProcesses.clear();
+
+ // ACCUMULO-2985 stop the ExecutorService after we finished using it to stop accumulo procs
+ if (null != executor) {
+ List<Runnable> tasksRemaining = executor.shutdownNow();
+
+ // the single thread executor shouldn't have any pending tasks, but check anyways
+ if (!tasksRemaining.isEmpty()) {
+ log.warn("Unexpectedly had " + tasksRemaining.size() + " task(s) remaining in threadpool for execution when being stopped");
+ }
+
+ executor = null;
+ }
+
+ if (config.useMiniDFS() && miniDFS != null)
+ miniDFS.shutdown();
+ for (Process p : cleanup) {
+ p.destroy();
+ p.waitFor();
+ }
+ miniDFS = null;
+ }
+
+ /**
+ * @since 1.6.0
+ */
+ @Override
+ public MiniAccumuloConfigImpl getConfig() {
+ return config;
+ }
+
+ /**
+ * Utility method to get a connector to the MAC.
+ *
+ * @since 1.6.0
+ */
+ @Override
+ public Connector getConnector(String user, String passwd) throws AccumuloException, AccumuloSecurityException {
+ Instance instance = new ZooKeeperInstance(getClientConfig());
+ return instance.getConnector(user, new PasswordToken(passwd));
+ }
+
+ @Override
+ public ClientConfiguration getClientConfig() {
+ return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(
+ this.getZooKeepers());
+ }
+
+ public FileSystem getFileSystem() {
+ try {
+ return FileSystem.get(new URI(dfsUri), new Configuration());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // Visible for testing
+ protected void setShutdownExecutor(ExecutorService svc) {
+ this.executor = svc;
+ }
+
+ // Visible for testing
+ protected ExecutorService getShutdownExecutor() {
+ return executor;
+ }
+
+ private int stopProcessWithTimeout(final Process proc, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ FutureTask<Integer> future = new FutureTask<Integer>(new Callable<Integer>() {
+ @Override
+ public Integer call() throws InterruptedException {
+ proc.destroy();
+ return proc.waitFor();
+ }
+ });
+
+ executor.execute(future);
+
+ return future.get(timeout, unit);
+ }
+
+ /**
+ * Get programmatic interface to information available in a normal monitor. XXX the returned structure won't contain information about the metadata table
+ * until there is data in it. e.g. if you want to see the metadata table you should create a table.
+ *
+ * @since 1.6.1
+ */
+ public MasterMonitorInfo getMasterMonitorInfo() throws AccumuloException, AccumuloSecurityException {
+ MasterClientService.Iface client = null;
+ MasterMonitorInfo stats = null;
+ try {
+ Instance instance = new ZooKeeperInstance(getClientConfig());
+ client = MasterClient.getConnectionWithRetry(instance);
+ stats = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get(instance).toThrift(instance));
+ } catch (ThriftSecurityException exception) {
+ throw new AccumuloSecurityException(exception);
+ } catch (TException exception) {
+ throw new AccumuloException(exception);
+ } finally {
+ if (client != null) {
+ MasterClient.close(client);
+ }
+ }
+ return stats;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
index be29dbb,48e2906..30da0cba
--- a/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
@@@ -16,6 -16,6 +16,8 @@@
*/
package org.apache.accumulo.proxy;
++import static com.google.common.base.Charsets.UTF_8;
++
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
@@@ -194,13 -172,11 +195,13 @@@ public class ProxyServer implements Acc
}
protected Connector getConnector(ByteBuffer login) throws Exception {
- String[] pair = new String(login.array(), login.position(), login.remaining(), Constants.UTF8).split(",", 2);
- TCredentials user = CredentialHelper.fromByteArray(ByteBufferUtil.toBytes(login));
- if (user == null)
- throw new org.apache.accumulo.proxy.thrift.AccumuloSecurityException("unknown user");
- Connector connector = instance.getConnector(user.getPrincipal(), CredentialHelper.extractToken(user));
- return connector;
++ String[] pair = new String(login.array(), login.position(), login.remaining(), UTF_8).split(",", 2);
+ if (instance.getInstanceID().equals(pair[0])) {
+ Credentials creds = Credentials.deserialize(pair[1]);
+ return instance.getConnector(creds.getPrincipal(), creds.getToken());
+ } else {
+ throw new org.apache.accumulo.core.client.AccumuloSecurityException(pair[0], org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.INVALID_INSTANCEID);
+ }
}
private void handleAccumuloException(AccumuloException e) throws org.apache.accumulo.proxy.thrift.TableNotFoundException,
@@@ -1489,7 -1432,8 +1490,7 @@@
public ByteBuffer login(String principal, Map<String,String> loginProperties) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException {
try {
AuthenticationToken token = getToken(principal, loginProperties);
- ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(Constants.UTF8));
- TCredentials credential = CredentialHelper.create(principal, token, instance.getInstanceID());
- ByteBuffer login = ByteBuffer.wrap(CredentialHelper.asByteArray(credential));
++ ByteBuffer login = ByteBuffer.wrap((instance.getInstanceID() + "," + new Credentials(principal, token).serialize()).getBytes(UTF_8));
getConnector(login); // check to make sure user exists
return login;
} catch (AccumuloSecurityException e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------
[50/50] [abbrv] git commit: Merge branch '1.6'
Posted by ct...@apache.org.
Merge branch '1.6'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cfbdef13
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cfbdef13
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cfbdef13
Branch: refs/heads/master
Commit: cfbdef1345388767e81a05a154dac22040ad5283
Parents: 64d1bbe 9b20a9d
Author: Christopher Tubbs <ct...@apache.org>
Authored: Sat Nov 1 00:54:59 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Sat Nov 1 00:54:59 2014 -0400
----------------------------------------------------------------------
assemble/bin/bootstrap_config.sh | 2 +-
.../org/apache/accumulo/core/Constants.java | 5 ++-
.../apache/accumulo/core/cli/ClientOpts.java | 7 ++--
.../accumulo/core/client/BatchWriterConfig.java | 11 ++---
.../accumulo/core/client/ZooKeeperInstance.java | 14 +++----
.../client/impl/InstanceOperationsImpl.java | 4 +-
.../accumulo/core/client/impl/Namespaces.java | 7 ++--
.../core/client/impl/ReplicationClient.java | 4 +-
.../accumulo/core/client/impl/ServerClient.java | 4 +-
.../core/client/impl/TableOperationsImpl.java | 32 +++++++--------
.../accumulo/core/client/impl/Tables.java | 10 ++---
.../accumulo/core/client/impl/Writer.java | 5 +--
.../core/client/lexicoder/StringLexicoder.java | 6 +--
.../client/security/tokens/PasswordToken.java | 5 ++-
.../accumulo/core/conf/ConfigurationDocGen.java | 9 ++--
.../accumulo/core/data/ArrayByteSequence.java | 7 ++--
.../org/apache/accumulo/core/data/Column.java | 7 ++--
.../apache/accumulo/core/data/Condition.java | 8 ++--
.../org/apache/accumulo/core/data/Value.java | 4 +-
.../accumulo/core/file/BloomFilterLayer.java | 7 ++--
.../core/iterators/ColumnFamilyCounter.java | 5 ++-
.../accumulo/core/iterators/LongCombiner.java | 7 ++--
.../accumulo/core/iterators/conf/ColumnSet.java | 7 ++--
.../core/iterators/user/BigDecimalCombiner.java | 7 ++--
.../core/iterators/user/GrepIterator.java | 5 ++-
.../iterators/user/IntersectingIterator.java | 7 ++--
.../core/iterators/user/LargeRowFilter.java | 5 ++-
.../core/iterators/user/RegExFilter.java | 7 ++--
.../iterators/user/RowDeletingIterator.java | 5 ++-
.../iterators/user/SummingArrayCombiner.java | 7 ++--
.../iterators/user/TransformingIterator.java | 7 ++--
.../core/iterators/user/VisibilityFilter.java | 5 ++-
.../core/replication/ReplicationSchema.java | 6 +--
.../core/replication/ReplicationTarget.java | 6 +--
.../accumulo/core/security/Authorizations.java | 12 +++---
.../core/security/ColumnVisibility.java | 43 ++++++++++----------
.../accumulo/core/security/Credentials.java | 12 +++---
.../core/security/VisibilityConstraint.java | 5 ++-
.../core/security/VisibilityParseException.java | 5 ++-
.../accumulo/core/trace/ZooTraceClient.java | 5 ++-
.../apache/accumulo/core/util/ByteArraySet.java | 5 ++-
.../accumulo/core/util/ByteBufferUtil.java | 5 ++-
.../apache/accumulo/core/util/CreateToken.java | 6 +--
.../org/apache/accumulo/core/util/Encoding.java | 4 +-
.../apache/accumulo/core/util/FastFormat.java | 4 +-
.../org/apache/accumulo/core/util/Merge.java | 5 ++-
.../apache/accumulo/core/util/MonitorUtil.java | 4 +-
.../org/apache/accumulo/core/util/TextUtil.java | 6 +--
.../core/client/BatchWriterConfigTest.java | 8 ++--
.../core/client/ZooKeeperInstanceTest.java | 10 ++---
.../core/client/mock/MockNamespacesTest.java | 4 +-
.../tokens/CredentialProviderTokenTest.java | 8 ++--
.../security/tokens/PasswordTokenTest.java | 6 +--
.../accumulo/core/data/ConditionTest.java | 13 +++---
.../core/data/ConditionalMutationTest.java | 10 ++---
.../apache/accumulo/core/data/ValueTest.java | 5 ++-
.../core/iterators/user/RegExFilterTest.java | 7 ++--
.../core/security/ColumnVisibilityTest.java | 6 +--
.../accumulo/core/security/CredentialsTest.java | 7 ++--
.../core/security/VisibilityConstraintTest.java | 4 +-
.../security/crypto/BlockedIOStreamTest.java | 10 ++---
.../ShardedTableDistributionFormatterTest.java | 8 ++--
.../org/apache/accumulo/fate/AdminUtil.java | 8 ++--
.../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++----
.../zookeeper/DistributedReadWriteLock.java | 18 ++++----
.../accumulo/fate/zookeeper/ZooCache.java | 8 ++--
.../apache/accumulo/fate/zookeeper/ZooLock.java | 6 +--
.../accumulo/fate/zookeeper/ZooReservation.java | 9 ++--
.../accumulo/fate/zookeeper/ZooSession.java | 6 +--
.../core/client/mapreduce/RangeInputSplit.java | 7 ++--
.../mapreduce/lib/impl/ConfiguratorBase.java | 5 +--
.../mapreduce/lib/impl/InputConfigurator.java | 16 ++++----
.../mapreduce/lib/impl/OutputConfigurator.java | 8 ++--
.../lib/partition/RangePartitioner.java | 7 ++--
.../impl/MiniAccumuloClusterImpl.java | 5 ++-
.../org/apache/accumulo/proxy/ProxyServer.java | 7 ++--
.../apache/accumulo/proxy/TestProxyClient.java | 15 +++----
.../java/org/apache/accumulo/proxy/Util.java | 5 ++-
.../org/apache/accumulo/server/Accumulo.java | 5 ++-
.../accumulo/server/client/HdfsZooInstance.java | 11 ++---
.../server/conf/ZooCachePropertyAccessor.java | 5 ++-
.../accumulo/server/conf/ZooConfiguration.java | 5 ++-
.../server/constraints/MetadataConstraints.java | 15 +++----
.../apache/accumulo/server/init/Initialize.java | 16 ++++----
.../accumulo/server/master/LiveTServerSet.java | 4 +-
.../server/master/state/DeadServerList.java | 7 ++--
.../server/master/state/TServerInstance.java | 8 ++--
.../master/state/TabletStateChangeIterator.java | 5 ++-
.../accumulo/server/master/state/ZooStore.java | 5 ++-
.../master/state/ZooTabletStateStore.java | 11 ++---
.../server/metrics/AbstractMetricsImpl.java | 5 ++-
.../accumulo/server/monitor/LogService.java | 5 ++-
.../server/replication/ReplicationUtil.java | 5 ++-
.../server/security/SecurityOperation.java | 5 ++-
.../server/security/SystemCredentials.java | 11 ++---
.../security/handler/ZKAuthenticator.java | 5 ++-
.../server/security/handler/ZKAuthorizor.java | 5 ++-
.../server/security/handler/ZKPermHandler.java | 5 ++-
.../accumulo/server/tables/TableManager.java | 19 +++++----
.../server/tablets/UniqueNameAllocator.java | 11 ++---
.../server/util/AddFilesWithMissingEntries.java | 5 ++-
.../accumulo/server/util/ChangeSecret.java | 5 ++-
.../accumulo/server/util/CleanZookeeper.java | 8 ++--
.../accumulo/server/util/DeleteZooInstance.java | 7 ++--
.../accumulo/server/util/DumpZookeeper.java | 6 +--
.../accumulo/server/util/ListInstances.java | 7 ++--
.../server/util/MasterMetadataUtil.java | 13 +++---
.../accumulo/server/util/MetadataTableUtil.java | 33 +++++++--------
.../accumulo/server/util/NamespacePropUtil.java | 4 +-
.../accumulo/server/util/RandomizeVolumes.java | 4 +-
.../accumulo/server/util/RestoreZookeeper.java | 7 ++--
.../accumulo/server/util/SendLogToChainsaw.java | 9 ++--
.../accumulo/server/util/SystemPropUtil.java | 4 +-
.../accumulo/server/util/TablePropUtil.java | 4 +-
.../accumulo/server/util/TabletServerLocks.java | 5 ++-
.../server/watcher/MonitorLog4jWatcher.java | 4 +-
.../server/zookeeper/DistributedWorkQueue.java | 7 ++--
.../accumulo/server/zookeeper/ZooQueueLock.java | 9 ++--
.../server/zookeeper/ZooReaderWriter.java | 4 +-
.../zookeeper/ZooReaderWriterFactory.java | 4 +-
.../server/conf/NamespaceConfigurationTest.java | 31 +++++++-------
.../conf/ServerConfigurationFactoryTest.java | 5 +--
.../server/conf/TableConfigurationTest.java | 10 ++---
.../conf/ZooCachePropertyAccessorTest.java | 4 +-
.../java/org/apache/accumulo/master/Master.java | 15 +++----
.../master/recovery/RecoveryManager.java | 5 ++-
.../MasterReplicationCoordinator.java | 5 ++-
.../accumulo/master/state/SetGoalState.java | 4 +-
.../accumulo/master/tableOps/BulkImport.java | 9 ++--
.../master/tableOps/CancelCompactions.java | 10 ++---
.../accumulo/master/tableOps/CompactRange.java | 15 +++----
.../accumulo/master/tableOps/ExportTable.java | 9 ++--
.../accumulo/master/tableOps/ImportTable.java | 17 ++++----
.../accumulo/master/tableOps/RenameTable.java | 6 +--
.../apache/accumulo/master/tableOps/Utils.java | 15 +++----
.../master/tserverOps/ShutdownTServer.java | 4 +-
.../org/apache/accumulo/monitor/Monitor.java | 7 ++--
.../accumulo/monitor/ZooKeeperStatus.java | 7 ++--
.../accumulo/monitor/servlets/BasicServlet.java | 11 ++---
.../monitor/servlets/DefaultServlet.java | 5 ++-
.../accumulo/monitor/servlets/ShellServlet.java | 5 ++-
.../accumulo/monitor/servlets/XMLServlet.java | 7 ++--
.../accumulo/monitor/servlets/trace/Basic.java | 5 ++-
.../org/apache/accumulo/tracer/TraceServer.java | 9 ++--
.../tserver/BulkFailedCopyProcessor.java | 6 +--
.../apache/accumulo/tserver/TabletServer.java | 12 +++---
.../apache/accumulo/tserver/log/DfsLogger.java | 16 ++++----
.../accumulo/tserver/logger/LogFileValue.java | 11 ++---
.../accumulo/tserver/logger/LogReader.java | 5 ++-
.../replication/ReplicationProcessor.java | 5 ++-
.../apache/accumulo/tserver/tablet/Tablet.java | 11 ++---
.../replication/ReplicationProcessorTest.java | 5 ++-
.../java/org/apache/accumulo/shell/Shell.java | 10 +++--
.../org/apache/accumulo/shell/ShellUtil.java | 7 ++--
.../shell/commands/AuthenticateCommand.java | 5 ++-
.../shell/commands/ExecfileCommand.java | 5 ++-
.../accumulo/shell/commands/HiddenCommand.java | 5 ++-
.../accumulo/shell/commands/PasswdCommand.java | 6 +--
.../shell/commands/QuotedStringTokenizer.java | 5 ++-
.../accumulo/shell/commands/UserCommand.java | 5 ++-
.../apache/accumulo/shell/mock/MockShell.java | 8 ++--
.../apache/accumulo/shell/ShellUtilTest.java | 4 +-
.../shell/format/DeleterFormatterTest.java | 12 +++---
.../accumulo/test/BulkImportDirectory.java | 5 ++-
.../accumulo/test/NativeMapConcurrencyTest.java | 5 ++-
.../accumulo/test/NativeMapPerformanceTest.java | 7 ++--
.../accumulo/test/NativeMapStressTest.java | 5 ++-
.../apache/accumulo/test/TestBinaryRows.java | 7 ++--
.../org/apache/accumulo/test/TestIngest.java | 7 ++--
.../accumulo/test/TestMultiTableIngest.java | 5 ++-
.../test/continuous/ContinuousIngest.java | 9 ++--
.../test/continuous/ContinuousMoru.java | 7 ++--
.../test/continuous/ContinuousQuery.java | 5 ++-
.../test/continuous/ContinuousScanner.java | 5 ++-
.../test/continuous/ContinuousVerify.java | 7 ++--
.../test/continuous/ContinuousWalk.java | 17 ++++----
.../accumulo/test/continuous/Histogram.java | 5 ++-
.../test/continuous/PrintScanTimeHistogram.java | 5 ++-
.../accumulo/test/continuous/TimeBinner.java | 5 ++-
.../test/continuous/UndefinedAnalyzer.java | 9 ++--
.../test/functional/CacheTestReader.java | 7 ++--
.../test/functional/CacheTestWriter.java | 11 ++---
.../accumulo/test/functional/ZombieTServer.java | 5 ++-
.../metadata/MetadataBatchScanTest.java | 7 ++--
.../apache/accumulo/test/randomwalk/Module.java | 7 ++--
.../test/randomwalk/bulk/BulkMinusOne.java | 4 +-
.../accumulo/test/randomwalk/bulk/Verify.java | 5 ++-
.../test/randomwalk/concurrent/BatchWrite.java | 5 ++-
.../test/randomwalk/concurrent/BulkImport.java | 5 ++-
.../concurrent/ChangeAuthorizations.java | 7 ++--
.../randomwalk/concurrent/StopTabletServer.java | 5 ++-
.../accumulo/test/randomwalk/image/Verify.java | 5 ++-
.../accumulo/test/randomwalk/image/Write.java | 9 ++--
.../test/randomwalk/multitable/Write.java | 5 ++-
.../randomwalk/security/SecurityHelper.java | 7 ++--
.../test/randomwalk/security/TableOp.java | 9 ++--
.../test/randomwalk/sequential/Write.java | 5 ++-
.../test/randomwalk/shard/BulkInsert.java | 5 ++-
.../test/randomwalk/shard/ExportIndex.java | 5 ++-
.../accumulo/test/scalability/Ingest.java | 5 ++-
.../apache/accumulo/test/AuditMessageIT.java | 6 +--
.../apache/accumulo/test/LargeSplitRowIT.java | 7 ++--
.../org/apache/accumulo/test/NamespacesIT.java | 8 ++--
.../java/org/apache/accumulo/test/VolumeIT.java | 6 +--
.../accumulo/test/functional/AddSplitIT.java | 5 ++-
.../test/functional/BadIteratorMincIT.java | 6 +--
.../test/functional/BatchScanSplitIT.java | 7 ++--
.../test/functional/BatchWriterFlushIT.java | 5 ++-
.../accumulo/test/functional/BulkFileIT.java | 5 ++-
.../accumulo/test/functional/ConcurrencyIT.java | 7 ++--
.../accumulo/test/functional/ConstraintIT.java | 23 ++++++-----
.../test/functional/CreateAndUseIT.java | 5 ++-
.../test/functional/DeleteEverythingIT.java | 4 +-
.../test/functional/DeleteRowsSplitIT.java | 6 +--
.../accumulo/test/functional/LargeRowIT.java | 7 ++--
.../accumulo/test/functional/NativeMapIT.java | 16 ++++----
.../test/functional/ScanIteratorIT.java | 7 ++--
.../accumulo/test/functional/ScanRangeIT.java | 5 ++-
.../test/functional/ScanSessionTimeOutIT.java | 5 ++-
.../test/functional/SplitRecoveryIT.java | 4 +-
.../accumulo/test/functional/TabletIT.java | 4 +-
.../accumulo/test/functional/VisibilityIT.java | 11 ++---
.../replication/MultiTserverReplicationIT.java | 7 ++--
.../UnusedWalDoesntCloseReplicationStatus.java | 6 +--
.../instrument/receivers/ZooSpanClient.java | 6 +--
225 files changed, 928 insertions(+), 790 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/assemble/bin/bootstrap_config.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index e7162c2,9897262..0229d4e
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -16,7 -16,9 +16,8 @@@
*/
package org.apache.accumulo.core;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
-import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
index e582160,041c9e6..10a5e08
--- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.cli;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
index 23ab39e,24ad86d..842a713
--- a/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/BatchWriterConfig.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.client;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@@ -203,12 -180,10 +204,12 @@@ public class BatchWriterConfig implemen
addField(fields, "maxWriteThreads", maxWriteThreads);
if (timeout != null)
addField(fields, "timeout", timeout);
+ if (durability != Durability.DEFAULT)
+ addField(fields, "durability", durability);
String output = StringUtils.join(",", fields);
- byte[] bytes = output.getBytes(StandardCharsets.UTF_8);
- byte[] len = String.format("%6s#", Integer.toString(bytes.length, 36)).getBytes(StandardCharsets.UTF_8);
+ byte[] bytes = output.getBytes(UTF_8);
+ byte[] len = String.format("%6s#", Integer.toString(bytes.length, 36)).getBytes(UTF_8);
if (len.length != 7)
throw new IllegalStateException("encoded length does not match expected value");
out.write(len);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 3dba358,a9c3292..4e11a14
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -16,10 -16,9 +16,10 @@@
*/
package org.apache.accumulo.core.client;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
index 1a9af63,6925f18..30c15ed
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
@@@ -16,9 -16,8 +16,9 @@@
*/
package org.apache.accumulo.core.client.impl;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
index 6be1a12,f32940c..3c2bc45
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.client.impl;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.security.SecurityPermission;
import java.util.LinkedList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
index 937dc7c,0000000..901128b
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
@@@ -1,246 -1,0 +1,246 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
- import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.net.HostAndPort;
+
+public class ReplicationClient {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationClient.class);
+
+ /**
+ * @param instance
+ * Instance for the peer replicant
+ * @return Client to the ReplicationCoordinator service
+ */
+ public static ReplicationCoordinator.Client getCoordinatorConnectionWithRetry(Instance instance) throws AccumuloException {
+ checkNotNull(instance);
+
+ for (int attempts = 1; attempts <= 10; attempts++) {
+
+ ReplicationCoordinator.Client result = getCoordinatorConnection(instance);
+ if (result != null)
+ return result;
+ log.debug("Could not get ReplicationCoordinator connection to {}, will retry", instance.getInstanceName());
+ try {
+ Thread.sleep(attempts * 250);
+ } catch (InterruptedException e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ throw new AccumuloException("Timed out trying to communicate with master from " + instance.getInstanceName());
+ }
+
+ public static ReplicationCoordinator.Client getCoordinatorConnection(Instance instance) {
+ List<String> locations = instance.getMasterLocations();
+
+ if (locations.size() == 0) {
+ log.debug("No masters for replication to instance {}", instance.getInstanceName());
+ return null;
+ }
+
+ // This is the master thrift service, we just want the hostname, not the port
+ String masterThriftService = locations.get(0);
+ if (masterThriftService.endsWith(":0")) {
+ log.warn("Master found for {} did not have real location {}", instance.getInstanceName(), masterThriftService);
+ return null;
+ }
+
+
+ AccumuloConfiguration conf = ClientConfigurationHelper.getClientRpcConfiguration(instance);
+
+ String zkPath = ZooUtil.getRoot(instance) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR;
+ String replCoordinatorAddr;
+
+ log.debug("Using ZooKeeper quorum at {} with path {} to find peer Master information", instance.getZooKeepers(), zkPath);
+
+ // Get the coordinator port for the master we're trying to connect to
+ try {
+ ZooReader reader = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
- replCoordinatorAddr = new String(reader.getData(zkPath, null), StandardCharsets.UTF_8);
++ replCoordinatorAddr = new String(reader.getData(zkPath, null), UTF_8);
+ } catch (KeeperException | InterruptedException e) {
+ log.error("Could not fetch remote coordinator port", e);
+ return null;
+ }
+
+ // Throw the hostname and port through HostAndPort to get some normalization
+ HostAndPort coordinatorAddr = HostAndPort.fromString(replCoordinatorAddr);
+
+ log.debug("Connecting to master at {}", coordinatorAddr);
+
+ try {
+ // Master requests can take a long time: don't ever time out
+ ReplicationCoordinator.Client client = ThriftUtil.getClientNoTimeout(new ReplicationCoordinator.Client.Factory(), coordinatorAddr.toString(),
+ conf);
+ return client;
+ } catch (TTransportException tte) {
+ log.debug("Failed to connect to master coordinator service ({})", coordinatorAddr, tte);
+ return null;
+ }
+ }
+
+ /**
+ * Attempt a single time to create a ReplicationServicer client to the given host
+ *
+ * @param inst
+ * Instance to the peer replicant
+ * @param server
+ * Server to connect to
+ * @return A ReplicationServicer client to the given host in the given instance
+ */
+ public static ReplicationServicer.Client getServicerConnection(Instance inst, String server) throws TTransportException {
+ checkNotNull(inst);
+ checkNotNull(server);
+
+ try {
+ return ThriftUtil.getClientNoTimeout(new ReplicationServicer.Client.Factory(), server, ClientConfigurationHelper.getClientRpcConfiguration(inst));
+ } catch (TTransportException tte) {
+ log.debug("Failed to connect to servicer ({}), will retry...", server, tte);
+ throw tte;
+ }
+ }
+
+ public static void close(ReplicationCoordinator.Iface iface) {
+ TServiceClient client = (TServiceClient) iface;
+ if (client != null && client.getInputProtocol() != null && client.getInputProtocol().getTransport() != null) {
+ ThriftTransportPool.getInstance().returnTransport(client.getInputProtocol().getTransport());
+ } else {
+ log.debug("Attempt to close null connection to the remote system", new Exception());
+ }
+ }
+
+ public static void close(ReplicationServicer.Iface iface) {
+ TServiceClient client = (TServiceClient) iface;
+ if (client != null && client.getInputProtocol() != null && client.getInputProtocol().getTransport() != null) {
+ ThriftTransportPool.getInstance().returnTransport(client.getInputProtocol().getTransport());
+ } else {
+ log.debug("Attempt to close null connection to the remote system", new Exception());
+ }
+ }
+
+ public static <T> T executeCoordinatorWithReturn(Instance instance, ClientExecReturn<T,ReplicationCoordinator.Client> exec) throws AccumuloException,
+ AccumuloSecurityException {
+ ReplicationCoordinator.Client client = null;
+ for (int i = 0; i < 10; i++) {
+ try {
+ client = getCoordinatorConnectionWithRetry(instance);
+ return exec.execute(client);
+ } catch (TTransportException tte) {
+ log.debug("ReplicationClient coordinator request failed, retrying ... ", tte);
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new AccumuloException(e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+
+ throw new AccumuloException("Could not connect to ReplicationCoordinator at " + instance.getInstanceName());
+ }
+
+ public static void executeCoordinator(Instance instance, ClientExec<ReplicationCoordinator.Client> exec) throws AccumuloException, AccumuloSecurityException {
+ ReplicationCoordinator.Client client = null;
+ try {
+ client = getCoordinatorConnectionWithRetry(instance);
+ exec.execute(client);
+ } catch (TTransportException tte) {
+ log.debug("ReplicationClient coordinator request failed, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+
+ public static <T> T executeServicerWithReturn(Instance instance, String tserver, ClientExecReturn<T,ReplicationServicer.Client> exec)
+ throws AccumuloException, AccumuloSecurityException, TTransportException {
+ ReplicationServicer.Client client = null;
+ while (true) {
+ try {
+ client = getServicerConnection(instance, tserver);
+ return exec.execute(client);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+ }
+
+ public static void executeServicer(Instance instance, String tserver, ClientExec<ReplicationServicer.Client> exec) throws AccumuloException,
+ AccumuloSecurityException, TTransportException {
+ ReplicationServicer.Client client = null;
+ try {
+ client = getServicerConnection(instance, tserver);
+ exec.execute(client);
+ return;
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
index fcbb104,70bdbb0..2d76695
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
@@@ -16,9 -16,8 +16,9 @@@
*/
package org.apache.accumulo.core.client.impl;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import org.apache.accumulo.core.Constants;
@@@ -130,9 -130,10 +130,9 @@@ public class ServerClient
for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
byte[] data = ZooUtil.getLockData(zc, path);
- if (data != null && !new String(data, StandardCharsets.UTF_8).equals("master"))
+ if (data != null && !new String(data, UTF_8).equals("master"))
- servers.add(new ThriftTransportKey(
- new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT),
- rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance))));
+ servers.add(new ThriftTransportKey(new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT), rpcTimeout, SslConnectionParams
+ .forClient(ClientConfigurationHelper.getClientRpcConfiguration(instance))));
}
boolean opened = false;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 4e3d9c6,547ccc7..06b9e15
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@@ -16,7 -16,7 +16,8 @@@
*/
package org.apache.accumulo.core.client.impl;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.BufferedReader;
import java.io.IOException;
@@@ -212,10 -208,9 +212,10 @@@ public class TableOperationsImpl extend
*/
@Override
public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
- ArgumentChecker.notNull(tableName, timeType);
+ checkArgument(tableName != null, "tableName is null");
+ checkArgument(timeType != null, "timeType is null");
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(StandardCharsets.UTF_8)));
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(UTF_8)));
Map<String,String> opts;
if (limitVersion)
@@@ -537,9 -532,9 +537,9 @@@
@Override
public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- ArgumentChecker.notNull(tableName);
+ checkArgument(tableName != null, "tableName is null");
ByteBuffer EMPTY = ByteBuffer.allocate(0);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
end == null ? EMPTY : TextUtil.getByteBuffer(end));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -553,9 -548,9 +553,9 @@@
@Override
public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- ArgumentChecker.notNull(tableName);
+ checkArgument(tableName != null, "tableName is null");
ByteBuffer EMPTY = ByteBuffer.allocate(0);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
end == null ? EMPTY : TextUtil.getByteBuffer(end));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -677,9 -672,9 +677,9 @@@
*/
@Override
public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- ArgumentChecker.notNull(tableName);
+ checkArgument(tableName != null, "tableName is null");
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)));
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -1336,9 -1324,9 +1336,9 @@@
@Override
public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
- ArgumentChecker.notNull(tableName);
+ checkArgument(tableName != null, "tableName is null");
String tableId = Tables.getTableId(instance, tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(StandardCharsets.UTF_8)));
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -1368,9 -1356,9 +1368,9 @@@
*/
@Override
public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
- ArgumentChecker.notNull(tableName);
+ checkArgument(tableName != null, "tableName is null");
String tableId = Tables.getTableId(instance, tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(StandardCharsets.UTF_8)));
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
Map<String,String> opts = new HashMap<String,String>();
try {
@@@ -1535,10 -1521,9 +1535,10 @@@
@Override
public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
- ArgumentChecker.notNull(tableName, exportDir);
+ checkArgument(tableName != null, "tableName is null");
+ checkArgument(exportDir != null, "exportDir is null");
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(exportDir.getBytes(StandardCharsets.UTF_8)));
+ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
Map<String,String> opts = Collections.emptyMap();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
index 0b522ca,4de95c3..c2b8001
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
@@@ -16,9 -16,8 +16,9 @@@
*/
package org.apache.accumulo.core.client.impl;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
- import java.nio.charset.StandardCharsets;
import java.security.SecurityPermission;
import java.util.HashMap;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index e76e2d9,de7bcb2..a38822b
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@@ -16,9 -16,7 +16,8 @@@
*/
package org.apache.accumulo.core.client.impl;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
-
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
index b5694f2,bf365ef..165248b
--- a/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.client.lexicoder;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
/**
* This lexicoder encodes/decodes a given String to/from bytes without further processing. It can be combined with other encoders like the
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index 39cf8a0,4d73979..53a0bcf
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.client.security.tokens;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 51fef69,618ff7e..60d1996
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.conf;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@@ -324,9 -311,9 +325,9 @@@ class ConfigurationDocGen
int n;
try {
while ((n = data.read(buffer)) > 0)
- doc.print(new String(buffer, 0, n, StandardCharsets.UTF_8));
+ doc.print(new String(buffer, 0, n, UTF_8));
} catch (IOException e) {
- e.printStackTrace();
+ log.debug("Encountered IOException while reading InputStream in appendResource().", e);
return;
} finally {
try {
@@@ -363,11 -346,11 +364,11 @@@
*/
public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
if (args.length == 2 && args[0].equals("--generate-html")) {
- new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateHtml();
+ new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateHtml();
- } else if (args.length == 2 && args[0].equals("--generate-latex")) {
- new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateLaTeX();
+ } else if (args.length == 2 && args[0].equals("--generate-asciidoc")) {
- new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateAsciidoc();
++ new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateAsciidoc();
} else {
- throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
+ throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-asciidoc <filename>");
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index 2467585,437590f..954fef3
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@@ -16,13 -16,11 +16,14 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.Serializable;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
+/**
+ * An implementation of {@link ByteSequence} that uses a backing byte array.
+ */
public class ArrayByteSequence extends ByteSequence implements Serializable {
private static final long serialVersionUID = 1L;
@@@ -67,25 -47,10 +68,25 @@@
}
+ /**
+ * Creates a new sequence from the given string. The bytes are determined from
+ * the string using the default platform encoding.
+ *
+ * @param s string to represent as bytes
+ */
public ArrayByteSequence(String s) {
- this(s.getBytes(StandardCharsets.UTF_8));
+ this(s.getBytes(UTF_8));
}
+ /**
+ * Creates a new sequence based on a byte buffer. If the byte buffer has an
+ * array, that array (and the buffer's offset and limit) are used; otherwise,
+ * a new backing array is created and a relative bulk get is performed to
+ * transfer the buffer's contents (starting at its current position and
+ * not beyond its limit).
+ *
+ * @param buffer byte buffer
+ */
public ArrayByteSequence(ByteBuffer buffer) {
this.length = buffer.remaining();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/Column.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Column.java
index 59bb5d0,a29e412..f259912
--- a/core/src/main/java/org/apache/accumulo/core/data/Column.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Column.java
@@@ -16,14 -16,14 +16,13 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
--
import org.apache.accumulo.core.data.thrift.TColumn;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
@@@ -204,23 -158,11 +203,23 @@@ public class Column implements Writable
return columnVisibility;
}
+ /**
+ * Gets a string representation of this column. The family, qualifier, and
+ * visibility are interpreted as strings using the platform default encoding;
+ * nulls are interpreted as empty strings.
+ *
+ * @return string form of column
+ */
public String toString() {
- return new String(columnFamily == null ? new byte[0] : columnFamily, StandardCharsets.UTF_8) + ":" + new String(columnQualifier == null ? new byte[0] : columnQualifier, StandardCharsets.UTF_8) + ":"
- + new String(columnVisibility == null ? new byte[0] : columnVisibility, StandardCharsets.UTF_8);
+ return new String(columnFamily == null ? new byte[0] : columnFamily, UTF_8) + ":" + new String(columnQualifier == null ? new byte[0] : columnQualifier, UTF_8) + ":"
+ + new String(columnVisibility == null ? new byte[0] : columnVisibility, UTF_8);
}
+ /**
+ * Converts this column to Thrift.
+ *
+ * @return Thrift column
+ */
public TColumn toThrift() {
return new TColumn(columnFamily == null ? null : ByteBuffer.wrap(columnFamily), columnQualifier == null ? null : ByteBuffer.wrap(columnQualifier),
columnVisibility == null ? null : ByteBuffer.wrap(columnVisibility));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index 71088c2,e6e31eb..e4b089d
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -16,9 -16,8 +16,9 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
- import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
@@@ -42,20 -42,10 +42,20 @@@ public class Condition
private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+ /**
+ * Creates a new condition. The initial column value and timestamp are null,
+ * and the initial column visibility is empty. Characters in the column family
+ * and column qualifier are encoded as bytes in the condition using UTF-8.
+ *
+ * @param cf column family
+ * @param cq column qualifier
+ * @throws IllegalArgumentException if any argument is null
+ */
public Condition(CharSequence cf, CharSequence cq) {
- ArgumentChecker.notNull(cf, cq);
+ checkArgument(cf != null, "cf is null");
+ checkArgument(cq != null, "cq is null");
- this.cf = new ArrayByteSequence(cf.toString().getBytes(StandardCharsets.UTF_8));
- this.cq = new ArrayByteSequence(cq.toString().getBytes(StandardCharsets.UTF_8));
+ this.cf = new ArrayByteSequence(cf.toString().getBytes(UTF_8));
+ this.cq = new ArrayByteSequence(cq.toString().getBytes(UTF_8));
this.cv = EMPTY;
}
@@@ -146,17 -94,14 +146,17 @@@
}
/**
- * see {@link #setValue(byte[])}
+ * This method sets the expected value of a column. In order for the condition to pass the column must exist and have this value. If a value is not set, then
+ * the column must be absent for the condition to pass. The passed-in character sequence is encoded as UTF-8.
+ * See {@link #setValue(byte[])}.
*
- * @return returns this
+ * @param value value
+ * @return this condition
+ * @throws IllegalArgumentException if value is null
*/
-
public Condition setValue(CharSequence value) {
- ArgumentChecker.notNull(value);
+ checkArgument(value != null, "value is null");
- this.val = new ArrayByteSequence(value.toString().getBytes(StandardCharsets.UTF_8));
+ this.val = new ArrayByteSequence(value.toString().getBytes(UTF_8));
return this;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Value.java
index 7679dd3,d256107..48eae02
--- a/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@@ -17,15 -17,17 +17,15 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.util.ByteBufferUtil.toBytes;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.List;
-import com.google.common.base.Preconditions;
-
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
@@@ -225,12 -202,12 +225,12 @@@ public class Value implements WritableC
}
return false;
}
-
+
@Override
public String toString() {
- return new String(get(), StandardCharsets.UTF_8);
+ return new String(get(), UTF_8);
}
-
+
/**
* A Comparator optimized for Value.
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
index 1bc54f2,654689c..90506c3
--- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
@@@ -18,6 -18,8 +18,8 @@@
package org.apache.accumulo.core.file;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
index 4062c3a,f19a97c..b3607ef
--- a/core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/ColumnFamilyCounter.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
index 3c948c6,bc96e57..3c6c6d2
--- a/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.iterators;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
index ad43736,cecd87e..01ed523
--- a/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.iterators.conf;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
index ccf7d86,1f73e0d..a9cd774
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.math.BigDecimal;
- import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
index 2760914,1046480..e859363
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
index 37157ed,d541454..732a76c
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
index 47b3c77,47f3cfd..4db379b
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/LargeRowFilter.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
index 9aff04f,5b43838..8d56889
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.io.UnsupportedEncodingException;
- import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
index ee2e1ae,3f317ea..3c24f0e
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowDeletingIterator.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
index bcd085e,0b6c183..9bdc883
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 17e3c30,bab900e..6b51ac5
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
index 4a10511,91f9416..878aa8e
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.accumulo.core.client.IteratorSetting;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
index 6330dca,0000000..491fda5
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
@@@ -1,298 -1,0 +1,298 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.replication;
+
- import java.nio.charset.CharacterCodingException;
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
++import java.nio.charset.CharacterCodingException;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.client.lexicoder.ULongLexicoder;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ *
+ */
+public class ReplicationSchema {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationSchema.class);
+
+ /**
+ * Portion of a file that must be replication to the given target: peer and some identifying location on that peer, e.g. remote table ID
+ * <p>
+ * <code>hdfs://localhost:8020/accumulo/wal/tserver+port/WAL work:serialized_ReplicationTarget [] -> Status Protobuf</code>
+ */
+ public static class WorkSection {
+ public static final Text NAME = new Text("work");
+ private static final ByteSequence BYTE_SEQ_NAME = new ArrayByteSequence("work");
+
+ public static void getFile(Key k, Text buff) {
+ Preconditions.checkNotNull(k);
+ Preconditions.checkNotNull(buff);
+ Preconditions.checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication work key with incorrect colfam");
+ _getFile(k, buff);
+ }
+
+ public static ReplicationTarget getTarget(Key k) {
+ return getTarget(k, new Text());
+ }
+
+ public static ReplicationTarget getTarget(Key k, Text buff) {
+ Preconditions.checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication work key with incorrect colfam");
+ k.getColumnQualifier(buff);
+
+ return ReplicationTarget.from(buff);
+ }
+
+ /**
+ * Limit the scanner to only pull replication work records
+ */
+ public static void limit(ScannerBase scanner) {
+ scanner.fetchColumnFamily(NAME);
+ }
+
+ public static Mutation add(Mutation m, Text serializedTarget, Value v) {
+ m.put(NAME, serializedTarget, v);
+ return m;
+ }
+ }
+
+ /**
+ * Holds replication markers tracking status for files
+ * <p>
+ * <code>hdfs://localhost:8020/accumulo/wal/tserver+port/WAL repl:local_table_id [] -> Status Protobuf</code>
+ */
+ public static class StatusSection {
+ public static final Text NAME = new Text("repl");
+ private static final ByteSequence BYTE_SEQ_NAME = new ArrayByteSequence("repl");
+
+ /**
+ * Extract the table ID from the key (inefficiently if called repeatedly)
+ *
+ * @param k
+ * Key to extract from
+ * @return The table ID
+ * @see #getTableId(Key,Text)
+ */
+ public static String getTableId(Key k) {
+ Text buff = new Text();
+ getTableId(k, buff);
+ return buff.toString();
+ }
+
+ /**
+ * Extract the table ID from the key into the given {@link Text}
+ *
+ * @param k
+ * Key to extract from
+ * @param buff
+ * Text to place table ID into
+ */
+ public static void getTableId(Key k, Text buff) {
+ Preconditions.checkNotNull(k);
+ Preconditions.checkNotNull(buff);
+
+ k.getColumnQualifier(buff);
+ }
+
+ /**
+ * Extract the file name from the row suffix into the given {@link Text}
+ *
+ * @param k
+ * Key to extract from
+ * @param buff
+ * Text to place file name into
+ */
+ public static void getFile(Key k, Text buff) {
+ Preconditions.checkNotNull(k);
+ Preconditions.checkNotNull(buff);
+ Preconditions.checkArgument(BYTE_SEQ_NAME.equals(k.getColumnFamilyData()), "Given replication status key with incorrect colfam");
+
+ _getFile(k, buff);
+ }
+
+ /**
+ * Limit the scanner to only return Status records
+ */
+ public static void limit(ScannerBase scanner) {
+ scanner.fetchColumnFamily(NAME);
+ }
+
+ public static Mutation add(Mutation m, Text tableId, Value v) {
+ m.put(NAME, tableId, v);
+ return m;
+ }
+ }
+
+ /**
+ * Holds the order in which files needed for replication were closed. The intent is to be able to guarantee that files which were closed earlier were
+ * replicated first and we don't replay data in the wrong order on our peers
+ * <p>
+ * <code>encodedTimeOfClosure\x00hdfs://localhost:8020/accumulo/wal/tserver+port/WAL order:source_table_id [] -> Status Protobuf</code>
+ */
+ public static class OrderSection {
+ public static final Text NAME = new Text("order");
+ public static final Text ROW_SEPARATOR = new Text(new byte[]{0});
+ private static final ULongLexicoder longEncoder = new ULongLexicoder();
+
+ /**
+ * Extract the table ID from the given key (inefficiently if called repeatedly)
+ *
+ * @param k
+ * OrderSection Key
+ * @return source table id
+ */
+ public static String getTableId(Key k) {
+ Text buff = new Text();
+ getTableId(k, buff);
+ return buff.toString();
+ }
+
+ /**
+ * Extract the table ID from the given key
+ *
+ * @param k
+ * OrderSection key
+ * @param buff
+ * Text to place table ID into
+ */
+ public static void getTableId(Key k, Text buff) {
+ Preconditions.checkNotNull(k);
+ Preconditions.checkNotNull(buff);
+
+ k.getColumnQualifier(buff);
+ }
+
+ /**
+ * Limit the scanner to only return Order records
+ */
+ public static void limit(ScannerBase scanner) {
+ scanner.fetchColumnFamily(NAME);
+ }
+
+ /**
+ * Creates the Mutation for the Order section for the given file and time
+ *
+ * @param file
+ * Filename
+ * @param timeInMillis
+ * Time in millis that the file was closed
+ * @return Mutation for the Order section
+ */
+ public static Mutation createMutation(String file, long timeInMillis) {
+ Preconditions.checkNotNull(file);
+ Preconditions.checkArgument(timeInMillis >= 0, "timeInMillis must be greater than zero");
+
+ // Encode the time so it sorts properly
+ byte[] rowPrefix = longEncoder.encode(timeInMillis);
+ Text row = new Text(rowPrefix);
+
+ // Normalize the file using Path
+ Path p = new Path(file);
+ String pathString = p.toUri().toString();
+
+ log.trace("Normalized {} into {}", file, pathString);
+
+ // Append the file as a suffix to the row
- row.append((ROW_SEPARATOR + pathString).getBytes(StandardCharsets.UTF_8), 0, pathString.length() + ROW_SEPARATOR.getLength());
++ row.append((ROW_SEPARATOR + pathString).getBytes(UTF_8), 0, pathString.length() + ROW_SEPARATOR.getLength());
+
+ // Make the mutation and add the column update
+ return new Mutation(row);
+ }
+
+ /**
+ * Add a column update to the given mutation with the provided tableId and value
+ *
+ * @param m
+ * Mutation for OrderSection
+ * @param tableId
+ * Source table id
+ * @param v
+ * Serialized Status msg
+ * @return The original Mutation
+ */
+ public static Mutation add(Mutation m, Text tableId, Value v) {
+ m.put(NAME, tableId, v);
+ return m;
+ }
+
+ public static long getTimeClosed(Key k) {
+ return getTimeClosed(k, new Text());
+ }
+
+ public static long getTimeClosed(Key k, Text buff) {
+ k.getRow(buff);
+ int offset = 0;
+ // find the last offset
+ while (true) {
+ int nextOffset = buff.find(ROW_SEPARATOR.toString(), offset + 1);
+ if (-1 == nextOffset) {
+ break;
+ }
+ offset = nextOffset;
+ }
+
+ if (-1 == offset) {
+ throw new IllegalArgumentException("Row does not contain expected separator for OrderSection");
+ }
+
+ byte[] encodedLong = new byte[offset];
+ System.arraycopy(buff.getBytes(), 0, encodedLong, 0, offset);
+ return longEncoder.decode(encodedLong);
+ }
+
+ public static String getFile(Key k) {
+ Text buff = new Text();
+ return getFile(k, buff);
+ }
+
+ public static String getFile(Key k, Text buff) {
+ k.getRow(buff);
+ int offset = 0;
+ // find the last offset
+ while (true) {
+ int nextOffset = buff.find(ROW_SEPARATOR.toString(), offset + 1);
+ if (-1 == nextOffset) {
+ break;
+ }
+ offset = nextOffset;
+ }
+
+ if (-1 == offset) {
+ throw new IllegalArgumentException("Row does not contain expected separator for OrderSection");
+ }
+
+ try {
+ return Text.decode(buff.getBytes(), offset + 1, buff.getLength() - (offset + 1));
+ } catch (CharacterCodingException e) {
+ throw new IllegalArgumentException("Could not decode file path", e);
+ }
+ }
+ }
+
+ private static void _getFile(Key k, Text buff) {
+ k.getRow(buff);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
index f882a8d,0000000..9c57ad9
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
@@@ -1,188 -1,0 +1,188 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.replication;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
-
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Container for where some work needs to be replicated
+ */
+public class ReplicationTarget implements Writable {
+
+ private String peerName;
+ private String remoteIdentifier;
+ private String sourceTableId;
+
+ public ReplicationTarget() { }
+
+ public ReplicationTarget(String peerName, String remoteIdentifier, String sourceTableId) {
+ this.peerName = peerName;
+ this.remoteIdentifier = remoteIdentifier;
+ this.sourceTableId = sourceTableId;
+ }
+
+ public String getPeerName() {
+ return peerName;
+ }
+
+ public void setPeerName(String peerName) {
+ this.peerName = peerName;
+ }
+
+ public String getRemoteIdentifier() {
+ return remoteIdentifier;
+ }
+
+ public void setRemoteIdentifier(String remoteIdentifier) {
+ this.remoteIdentifier = remoteIdentifier;
+ }
+
+ public String getSourceTableId() {
+ return sourceTableId;
+ }
+
+ public void setSourceTableId(String sourceTableId) {
+ this.sourceTableId = sourceTableId;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ if (null == peerName) {
+ out.writeBoolean(false);
+ } else {
+ out.writeBoolean(true);
+ WritableUtils.writeString(out, peerName);
+ }
+
+ if (null == remoteIdentifier) {
+ out.writeBoolean(false);
+ } else {
+ out.writeBoolean(true);
+ WritableUtils.writeString(out, remoteIdentifier);
+ }
+
+ if (null == sourceTableId) {
+ out.writeBoolean(false);
+ } else {
+ out.writeBoolean(true);
+ WritableUtils.writeString(out, sourceTableId);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ if (in.readBoolean()) {
+ this.peerName = WritableUtils.readString(in);
+ }
+ if (in.readBoolean()) {
+ this.remoteIdentifier = WritableUtils.readString(in);
+ }
+ if (in.readBoolean()) {
+ this.sourceTableId = WritableUtils.readString(in);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(64);
+ sb.append("Remote Name: ").append(peerName).append(" Remote identifier: ").append(remoteIdentifier).append(" Source Table ID: ").append(sourceTableId);
+ return sb.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return peerName.hashCode() ^ remoteIdentifier.hashCode() ^ sourceTableId.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ReplicationTarget) {
+ ReplicationTarget other = (ReplicationTarget) o;
+
+ return peerName.equals(other.peerName) && remoteIdentifier.equals(other.remoteIdentifier) && sourceTableId.equals(other.sourceTableId);
+ }
+
+ return false;
+ }
+
+ /**
+ * Deserialize a ReplicationTarget
+ * @param t Serialized copy
+ * @return the deserialized version
+ */
+ public static ReplicationTarget from(Text t) {
+ ReplicationTarget target = new ReplicationTarget();
+ DataInputBuffer buffer = new DataInputBuffer();
+ buffer.reset(t.getBytes(), t.getLength());
+
+ try {
+ target.readFields(buffer);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return target;
+ }
+
+ /**
+ * Deserialize a ReplicationTarget
+ * @param s Serialized copy
+ * @return the deserialized version
+ */
+ public static ReplicationTarget from(String s) {
+ ReplicationTarget target = new ReplicationTarget();
+ DataInputBuffer buffer = new DataInputBuffer();
- buffer.reset(s.getBytes(StandardCharsets.UTF_8), s.length());
++ buffer.reset(s.getBytes(UTF_8), s.length());
+
+ try {
+ target.readFields(buffer);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return target;
+ }
+
+ /**
+ * Convenience method to serialize a ReplicationTarget to {@link Text} using the {@link Writable} methods without caring about
+ * performance penalties due to excessive object creation
+ * @return The serialized representation of the object
+ */
+ public Text toText() {
+ DataOutputBuffer buffer = new DataOutputBuffer();
+
+ try {
+ this.write(buffer);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ Text t = new Text();
+ // Throw it in a text for the mutation
+ t.set(buffer.getData(), 0, buffer.getLength());
+ return t;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
index 54ddcd9,7073d11..b6ce5b7
--- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
@@@ -16,7 -16,7 +16,8 @@@
*/
package org.apache.accumulo.core.security;
-import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.Serializable;
import java.nio.ByteBuffer;
@@@ -145,9 -145,9 +145,9 @@@ public class Authorizations implements
*/
public Authorizations(byte[] authorizations) {
- ArgumentChecker.notNull(authorizations);
+ checkArgument(authorizations != null, "authorizations is null");
- String authsString = new String(authorizations, StandardCharsets.UTF_8);
+ String authsString = new String(authorizations, UTF_8);
if (authsString.startsWith(HEADER)) {
// it's the new format
authsString = authsString.substring(HEADER.length());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index 152e067,e8f4e97..3e930f5
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.Serializable;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Credentials.java
index 7981ccf,0ecc31d..525a958
--- a/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
@@@ -16,9 -16,10 +16,9 @@@
*/
package org.apache.accumulo.core.security;
- import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ import java.nio.ByteBuffer;
-
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
index 1d29119,4f91dac..c4fe5ac
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.security;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
index 1d54ae4,178de91..ca24884
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.security;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.text.ParseException;
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
index 4c8b837,4c2f5c9..9586eaa
--- a/core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/trace/ZooTraceClient.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.trace;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
index fc3c114,b1047d9..4b7db82
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteArraySet.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
index 1a34628,8771832..dbf04dc
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index b70eef0,26a8c73..cfac8fe
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@@ -16,12 -16,13 +16,12 @@@
*/
package org.apache.accumulo.core.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
- import java.nio.charset.StandardCharsets;
--
import jline.console.ConsoleReader;
import org.apache.accumulo.core.cli.ClientOpts.Password;
[10/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index 9537f4e,0000000..83e83cf
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -1,1315 -1,0 +1,1317 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.ThriftTransportPool;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Iface;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Processor;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.master.thrift.MasterState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.AgeOffStore;
+import org.apache.accumulo.fate.Fate;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.master.recovery.RecoveryManager;
+import org.apache.accumulo.master.state.TableCounts;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
+import org.apache.accumulo.server.master.balancer.TabletBalancer;
+import org.apache.accumulo.server.master.state.CurrentState;
+import org.apache.accumulo.server.master.state.DeadServerList;
+import org.apache.accumulo.server.master.state.MergeInfo;
+import org.apache.accumulo.server.master.state.MergeState;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.RootTabletStateStore;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.accumulo.server.master.state.TabletState;
+import org.apache.accumulo.server.master.state.ZooStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.security.handler.ZKPermHandler;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.tables.TableObserver;
+import org.apache.accumulo.server.util.DefaultMap;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.RpcWrapper;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * The Master is responsible for assigning and balancing tablets to tablet servers.
+ *
+ * The master will also coordinate log recoveries and reports general status.
+ */
+public class Master implements LiveTServerSet.Listener, TableObserver, CurrentState {
+
+ final static Logger log = Logger.getLogger(Master.class);
+
+ final static int ONE_SECOND = 1000;
+ final private static Text METADATA_TABLE_ID = new Text(MetadataTable.ID);
+ final private static Text ROOT_TABLE_ID = new Text(RootTable.ID);
+ final static long TIME_TO_WAIT_BETWEEN_SCANS = 60 * ONE_SECOND;
+ final private static long TIME_BETWEEN_MIGRATION_CLEANUPS = 5 * 60 * ONE_SECOND;
+ final static long WAIT_BETWEEN_ERRORS = ONE_SECOND;
+ final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;
+ final private static int MAX_CLEANUP_WAIT_TIME = ONE_SECOND;
+ final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = ONE_SECOND;
+ final static int MAX_TSERVER_WORK_CHUNK = 5000;
+ final private static int MAX_BAD_STATUS_COUNT = 3;
+
+ final VolumeManager fs;
+ final private Instance instance;
+ final private String hostname;
+ final LiveTServerSet tserverSet;
+ final private List<TabletGroupWatcher> watchers = new ArrayList<TabletGroupWatcher>();
+ final SecurityOperation security;
+ final Map<TServerInstance,AtomicInteger> badServers = Collections.synchronizedMap(new DefaultMap<TServerInstance,AtomicInteger>(new AtomicInteger()));
+ final Set<TServerInstance> serversToShutdown = Collections.synchronizedSet(new HashSet<TServerInstance>());
+ final SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,TServerInstance>());
+ final EventCoordinator nextEvent = new EventCoordinator();
+ final private Object mergeLock = new Object();
+ RecoveryManager recoveryManager = null;
+
+ ZooLock masterLock = null;
+ private TServer clientService = null;
+ TabletBalancer tabletBalancer;
+
+ private MasterState state = MasterState.INITIAL;
+
+ Fate<Master> fate;
+
+ volatile SortedMap<TServerInstance,TabletServerStatus> tserverStatus = Collections.unmodifiableSortedMap(new TreeMap<TServerInstance,TabletServerStatus>());
+
+ synchronized MasterState getMasterState() {
+ return state;
+ }
+
+ public boolean stillMaster() {
+ return getMasterState() != MasterState.STOP;
+ }
+
+ static final boolean X = true;
+ static final boolean _ = false;
+ // @formatter:off
+ static final boolean transitionOK[][] = {
+ // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
+ /* INITIAL */ {X, X, _, _, _, _, X},
+ /* HAVE_LOCK */ {_, X, X, X, _, _, X},
+ /* SAFE_MODE */ {_, _, X, X, X, _, X},
+ /* NORMAL */ {_, _, X, X, X, _, X},
+ /* UNLOAD_METADATA_TABLETS */ {_, _, X, X, X, X, X},
+ /* UNLOAD_ROOT_TABLET */ {_, _, _, X, X, X, X},
+ /* STOP */ {_, _, _, _, _, X, X}};
+ //@formatter:on
+ synchronized void setMasterState(MasterState newState) {
+ if (state.equals(newState))
+ return;
+ if (!transitionOK[state.ordinal()][newState.ordinal()]) {
+ log.error("Programmer error: master should not transition from " + state + " to " + newState);
+ }
+ MasterState oldState = state;
+ state = newState;
+ nextEvent.event("State changed from %s to %s", oldState, newState);
+ if (newState == MasterState.STOP) {
+ // Give the server a little time before shutdown so the client
+ // thread requesting the stop can return
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ // This frees the main thread and will cause the master to exit
+ clientService.stop();
+ Master.this.nextEvent.event("stopped event loop");
+ }
+
+ }, 100l, 1000l);
+ }
+
+ if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {
+ upgradeZookeeper();
+ }
+
+ if (oldState != newState && (newState == MasterState.NORMAL)) {
+ upgradeMetadata();
+ }
+ }
+
+ private void moveRootTabletToRootTable(IZooReaderWriter zoo) throws Exception {
+ String dirZPath = ZooUtil.getRoot(instance) + RootTable.ZROOT_TABLET_PATH;
+
+ if (!zoo.exists(dirZPath)) {
+ Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet");
+ if (fs.exists(oldPath)) {
+ String newPath = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID;
+ fs.mkdirs(new Path(newPath));
+ if (!fs.rename(oldPath, new Path(newPath))) {
+ throw new IOException("Failed to move root tablet from " + oldPath + " to " + newPath);
+ }
+
+ log.info("Upgrade renamed " + oldPath + " to " + newPath);
+ }
+
+ Path location = null;
+
+ for (String basePath : ServerConstants.getTablesDirs()) {
+ Path path = new Path(basePath + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION);
+ if (fs.exists(path)) {
+ if (location != null) {
+ throw new IllegalStateException("Root table at multiple locations " + location + " " + path);
+ }
+
+ location = path;
+ }
+ }
+
+ if (location == null)
+ throw new IllegalStateException("Failed to find root tablet");
+
+ log.info("Upgrade setting root table location in zookeeper " + location);
+ zoo.putPersistentData(dirZPath, location.toString().getBytes(), NodeExistsPolicy.FAIL);
+ }
+ }
+
+ private boolean haveUpgradedZooKeeper = false;
+
+ private void upgradeZookeeper() {
+ // 1.5.1 and 1.6.0 both do some state checking after obtaining the zoolock for the
+ // monitor and before starting up. It's not tied to the data version at all (and would
+ // introduce unnecessary complexity to try to make the master do it), but be aware
+ // that the master is not the only thing that may alter zookeeper before starting.
+
+ final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
+ if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
+ // This Master hasn't started Fate yet, so any outstanding transactions must be from before the upgrade.
+ // Change to Guava's Verify once we use Guava 17.
+ if (null != fate) {
+ throw new IllegalStateException(
+ "Access to Fate should not have been initialized prior to the Master transitioning to active. Please save all logs and file a bug.");
+ }
+ Accumulo.abortIfFateTransactions();
+ try {
+ log.info("Upgrading zookeeper");
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ final String zooRoot = ZooUtil.getRoot(instance);
+
+ if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
+ log.debug("Handling updates for version " + ServerConstants.TWO_DATA_VERSIONS_AGO);
+
+ log.debug("Cleaning out remnants of logger role.");
+ zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
+
+ final byte[] zero = new byte[] {'0'};
+ log.debug("Initializing recovery area.");
+ zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
+
+ for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
+ log.debug("Prepping table " + id + " for compaction cancellations.");
+ zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
+ }
+ }
+
+ // create initial namespaces
+ String namespaces = ZooUtil.getRoot(instance) + Constants.ZNAMESPACES;
+ zoo.putPersistentData(namespaces, new byte[0], NodeExistsPolicy.SKIP);
+ for (Pair<String,String> namespace : Iterables.concat(
+ Collections.singleton(new Pair<String,String>(Namespaces.ACCUMULO_NAMESPACE, Namespaces.ACCUMULO_NAMESPACE_ID)),
+ Collections.singleton(new Pair<String,String>(Namespaces.DEFAULT_NAMESPACE, Namespaces.DEFAULT_NAMESPACE_ID)))) {
+ String ns = namespace.getFirst();
+ String id = namespace.getSecond();
+ log.debug("Upgrade creating namespace \"" + ns + "\" (ID: " + id + ")");
+ if (!Namespaces.exists(instance, id))
+ TableManager.prepareNewNamespaceState(instance.getInstanceID(), id, ns, NodeExistsPolicy.SKIP);
+ }
+
+ // create root table
+ log.debug("Upgrade creating table " + RootTable.NAME + " (ID: " + RootTable.ID + ")");
+ TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE,
+ NodeExistsPolicy.SKIP);
+ Initialize.initMetadataConfig();
+ // ensure root user can flush root table
+ security.grantTablePermission(SystemCredentials.get().toThrift(instance), security.getRootUsername(), RootTable.ID, TablePermission.ALTER_TABLE,
+ Namespaces.ACCUMULO_NAMESPACE_ID);
+
+ // put existing tables in the correct namespaces
+ String tables = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ for (String tableId : zoo.getChildren(tables)) {
+ String targetNamespace = (MetadataTable.ID.equals(tableId) || RootTable.ID.equals(tableId)) ? Namespaces.ACCUMULO_NAMESPACE_ID
+ : Namespaces.DEFAULT_NAMESPACE_ID;
- log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), Constants.UTF8) + " (ID: "
++ log.debug("Upgrade moving table " + new String(zoo.getData(tables + "/" + tableId + Constants.ZTABLE_NAME, null), UTF_8) + " (ID: "
+ + tableId + ") into namespace with ID " + targetNamespace);
- zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(Constants.UTF8), NodeExistsPolicy.SKIP);
++ zoo.putPersistentData(tables + "/" + tableId + Constants.ZTABLE_NAMESPACE, targetNamespace.getBytes(UTF_8), NodeExistsPolicy.SKIP);
+ }
+
+ // rename metadata table
+ log.debug("Upgrade renaming table " + MetadataTable.OLD_NAME + " (ID: " + MetadataTable.ID + ") to " + MetadataTable.NAME);
- zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(Constants.UTF8),
++ zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8),
+ NodeExistsPolicy.OVERWRITE);
+
+ moveRootTabletToRootTable(zoo);
+
+ // add system namespace permissions to existing users
+ ZKPermHandler perm = new ZKPermHandler();
+ perm.initialize(instance.getInstanceID(), true);
+ String users = ZooUtil.getRoot(instance) + "/users";
+ for (String user : zoo.getChildren(users)) {
+ zoo.putPersistentData(users + "/" + user + "/Namespaces", new byte[0], NodeExistsPolicy.SKIP);
+ perm.grantNamespacePermission(user, Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.READ);
+ }
+ perm.grantNamespacePermission("root", Namespaces.ACCUMULO_NAMESPACE_ID, NamespacePermission.ALTER_TABLE);
+ haveUpgradedZooKeeper = true;
+ } catch (Exception ex) {
+ log.fatal("Error performing upgrade", ex);
+ System.exit(1);
+ }
+ }
+ }
+
+ private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
+ private final CountDownLatch waitForMetadataUpgrade = new CountDownLatch(1);
+
+ private final ServerConfiguration serverConfig;
+
+ private void upgradeMetadata() {
+ // we make sure we're only doing the rest of this method once so that we can signal to other threads that an upgrade wasn't needed.
+ if (upgradeMetadataRunning.compareAndSet(false, true)) {
+ final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs);
+ if (Accumulo.persistentVersionNeedsUpgrade(accumuloPersistentVersion)) {
+ // sanity check that we passed the Fate verification prior to ZooKeeper upgrade, and that Fate still hasn't been started.
+ // Change both to use Guava's Verify once we use Guava 17.
+ if (!haveUpgradedZooKeeper) {
+ throw new IllegalStateException(
+ "We should only attempt to upgrade Accumulo's metadata table if we've already upgraded ZooKeeper. Please save all logs and file a bug.");
+ }
+ if (null != fate) {
+ throw new IllegalStateException(
+ "Access to Fate should not have been initialized prior to the Master finishing upgrades. Please save all logs and file a bug.");
+ }
+ Runnable upgradeTask = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ log.info("Starting to upgrade metadata table.");
+ if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
+ log.info("Updating Delete Markers in metadata table for version 1.4");
+ MetadataTableUtil.moveMetaDeleteMarkersFrom14(instance, SystemCredentials.get());
+ } else {
+ log.info("Updating Delete Markers in metadata table.");
+ MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get());
+ }
+ log.info("Updating persistent data version.");
+ Accumulo.updateAccumuloVersion(fs, accumuloPersistentVersion);
+ log.info("Upgrade complete");
+ waitForMetadataUpgrade.countDown();
+ } catch (Exception ex) {
+ log.fatal("Error performing upgrade", ex);
+ System.exit(1);
+ }
+
+ }
+ };
+
+ // need to run this in a separate thread because a lock is held that prevents metadata tablets from being assigned and this task writes to the
+ // metadata table
+ new Thread(upgradeTask).start();
+ } else {
+ waitForMetadataUpgrade.countDown();
+ }
+ }
+ }
+
+ private int assignedOrHosted(Text tableId) {
+ int result = 0;
+ for (TabletGroupWatcher watcher : watchers) {
+ TableCounts count = watcher.getStats(tableId);
+ result += count.hosted() + count.assigned();
+ }
+ return result;
+ }
+
+ private int totalAssignedOrHosted() {
+ int result = 0;
+ for (TabletGroupWatcher watcher : watchers) {
+ for (TableCounts counts : watcher.getStats().values()) {
+ result += counts.assigned() + counts.hosted();
+ }
+ }
+ return result;
+ }
+
+ private int nonMetaDataTabletsAssignedOrHosted() {
+ return totalAssignedOrHosted() - assignedOrHosted(new Text(MetadataTable.ID)) - assignedOrHosted(new Text(RootTable.ID));
+ }
+
+ private int notHosted() {
+ int result = 0;
+ for (TabletGroupWatcher watcher : watchers) {
+ for (TableCounts counts : watcher.getStats().values()) {
+ result += counts.assigned() + counts.assignedToDeadServers();
+ }
+ }
+ return result;
+ }
+
+ // The number of unassigned tablets that should be assigned: displayed on the monitor page
+ int displayUnassigned() {
+ int result = 0;
+ switch (getMasterState()) {
+ case NORMAL:
+ // Count offline tablets for online tables
+ for (TabletGroupWatcher watcher : watchers) {
+ TableManager manager = TableManager.getInstance();
+ for (Entry<Text,TableCounts> entry : watcher.getStats().entrySet()) {
+ Text tableId = entry.getKey();
+ TableCounts counts = entry.getValue();
+ TableState tableState = manager.getTableState(tableId.toString());
+ if (tableState != null && tableState.equals(TableState.ONLINE)) {
+ result += counts.unassigned() + counts.assignedToDeadServers() + counts.assigned();
+ }
+ }
+ }
+ break;
+ case SAFE_MODE:
+ // Count offline tablets for the metadata table
+ for (TabletGroupWatcher watcher : watchers) {
+ result += watcher.getStats(METADATA_TABLE_ID).unassigned();
+ }
+ break;
+ case UNLOAD_METADATA_TABLETS:
+ case UNLOAD_ROOT_TABLET:
+ for (TabletGroupWatcher watcher : watchers) {
+ result += watcher.getStats(METADATA_TABLE_ID).unassigned();
+ }
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+
+ public void mustBeOnline(final String tableId) throws ThriftTableOperationException {
+ Tables.clearCache(instance);
+ if (!Tables.getTableState(instance, tableId).equals(TableState.ONLINE))
+ throw new ThriftTableOperationException(tableId, null, TableOperation.MERGE, TableOperationExceptionType.OFFLINE, "table is not online");
+ }
+
+ public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
+ return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ }
+
+ private Master(ServerConfiguration config, VolumeManager fs, String hostname) throws IOException {
+ this.serverConfig = config;
+ this.instance = config.getInstance();
+ this.fs = fs;
+ this.hostname = hostname;
+
+ AccumuloConfiguration aconf = serverConfig.getConfiguration();
+
+ log.info("Version " + Constants.VERSION);
+ log.info("Instance " + instance.getInstanceID());
+ ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ security = AuditedSecurityOperation.getInstance();
+ tserverSet = new LiveTServerSet(instance, config.getConfiguration(), this);
+ this.tabletBalancer = aconf.instantiateClassProperty(Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
+ this.tabletBalancer.init(serverConfig);
+
+ try {
+ AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() {
+ @Override
+ public Iterator<Entry<String,String>> iterator() {
+ return getSystemConfiguration().iterator();
+ }
+ }));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ public TServerConnection getConnection(TServerInstance server) {
+ return tserverSet.getConnection(server);
+ }
+
+ public MergeInfo getMergeInfo(Text tableId) {
+ synchronized (mergeLock) {
+ try {
+ String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
+ if (!ZooReaderWriter.getInstance().exists(path))
+ return new MergeInfo();
+ byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat());
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(data, data.length);
+ MergeInfo info = new MergeInfo();
+ info.readFields(in);
+ return info;
+ } catch (KeeperException.NoNodeException ex) {
+ log.info("Error reading merge state, it probably just finished");
+ return new MergeInfo();
+ } catch (Exception ex) {
+ log.warn("Unexpected error reading merge state", ex);
+ return new MergeInfo();
+ }
+ }
+ }
+
+ public void setMergeState(MergeInfo info, MergeState state) throws IOException, KeeperException, InterruptedException {
+ synchronized (mergeLock) {
+ String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + info.getExtent().getTableId().toString() + "/merge";
+ info.setState(state);
+ if (state.equals(MergeState.NONE)) {
+ ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
+ } else {
+ DataOutputBuffer out = new DataOutputBuffer();
+ try {
+ info.write(out);
+ } catch (IOException ex) {
+ throw new RuntimeException("Unlikely", ex);
+ }
+ ZooReaderWriter.getInstance().putPersistentData(path, out.getData(),
+ state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE);
+ }
+ mergeLock.notifyAll();
+ }
+ nextEvent.event("Merge state of %s set to %s", info.getExtent(), state);
+ }
+
+ public void clearMergeState(Text tableId) throws IOException, KeeperException, InterruptedException {
+ synchronized (mergeLock) {
+ String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
+ ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP);
+ mergeLock.notifyAll();
+ }
+ nextEvent.event("Merge state of %s cleared", tableId);
+ }
+
+ void setMasterGoalState(MasterGoalState state) {
+ try {
+ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, state.name().getBytes(),
+ NodeExistsPolicy.OVERWRITE);
+ } catch (Exception ex) {
+ log.error("Unable to set master goal state in zookeeper");
+ }
+ }
+
+ MasterGoalState getMasterGoalState() {
+ while (true)
+ try {
+ byte[] data = ZooReaderWriter.getInstance().getData(ZooUtil.getRoot(instance) + Constants.ZMASTER_GOAL_STATE, null);
+ return MasterGoalState.valueOf(new String(data));
+ } catch (Exception e) {
+ log.error("Problem getting real goal state: " + e);
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public boolean hasCycled(long time) {
+ for (TabletGroupWatcher watcher : watchers) {
+ if (watcher.stats.lastScanFinished() < time)
+ return false;
+ }
+
+ return true;
+ }
+
+ public void clearMigrations(String tableId) {
+ synchronized (migrations) {
+ Iterator<KeyExtent> iterator = migrations.keySet().iterator();
+ while (iterator.hasNext()) {
+ KeyExtent extent = iterator.next();
+ if (extent.getTableId().toString().equals(tableId)) {
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+ static enum TabletGoalState {
+ HOSTED, UNASSIGNED, DELETED
+ };
+
+ TabletGoalState getSystemGoalState(TabletLocationState tls) {
+ switch (getMasterState()) {
+ case NORMAL:
+ return TabletGoalState.HOSTED;
+ case HAVE_LOCK: // fall-through intended
+ case INITIAL: // fall-through intended
+ case SAFE_MODE:
+ if (tls.extent.isMeta())
+ return TabletGoalState.HOSTED;
+ return TabletGoalState.UNASSIGNED;
+ case UNLOAD_METADATA_TABLETS:
+ if (tls.extent.isRootTablet())
+ return TabletGoalState.HOSTED;
+ return TabletGoalState.UNASSIGNED;
+ case UNLOAD_ROOT_TABLET:
+ return TabletGoalState.UNASSIGNED;
+ case STOP:
+ return TabletGoalState.UNASSIGNED;
+ default:
+ throw new IllegalStateException("Unknown Master State");
+ }
+ }
+
+ TabletGoalState getTableGoalState(KeyExtent extent) {
+ TableState tableState = TableManager.getInstance().getTableState(extent.getTableId().toString());
+ if (tableState == null)
+ return TabletGoalState.DELETED;
+ switch (tableState) {
+ case DELETING:
+ return TabletGoalState.DELETED;
+ case OFFLINE:
+ case NEW:
+ return TabletGoalState.UNASSIGNED;
+ default:
+ return TabletGoalState.HOSTED;
+ }
+ }
+
+ TabletGoalState getGoalState(TabletLocationState tls, MergeInfo mergeInfo) {
+ KeyExtent extent = tls.extent;
+ // Shutting down?
+ TabletGoalState state = getSystemGoalState(tls);
+ if (state == TabletGoalState.HOSTED) {
+ if (tls.current != null && serversToShutdown.contains(tls.current)) {
+ return TabletGoalState.UNASSIGNED;
+ }
+ // Handle merge transitions
+ if (mergeInfo.getExtent() != null) {
+ log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent));
+ if (mergeInfo.overlaps(extent)) {
+ switch (mergeInfo.getState()) {
+ case NONE:
+ case COMPLETE:
+ break;
+ case STARTED:
+ case SPLITTING:
+ return TabletGoalState.HOSTED;
+ case WAITING_FOR_CHOPPED:
+ if (tls.getState(onlineTabletServers()).equals(TabletState.HOSTED)) {
+ if (tls.chopped)
+ return TabletGoalState.UNASSIGNED;
+ } else {
+ if (tls.chopped && tls.walogs.isEmpty())
+ return TabletGoalState.UNASSIGNED;
+ }
+
+ return TabletGoalState.HOSTED;
+ case WAITING_FOR_OFFLINE:
+ case MERGING:
+ return TabletGoalState.UNASSIGNED;
+ }
+ }
+ }
+
+ // taking table offline?
+ state = getTableGoalState(extent);
+ if (state == TabletGoalState.HOSTED) {
+ // Maybe this tablet needs to be migrated
+ TServerInstance dest = migrations.get(extent);
+ if (dest != null && tls.current != null && !dest.equals(tls.current)) {
+ return TabletGoalState.UNASSIGNED;
+ }
+ }
+ }
+ return state;
+ }
+
+ private class MigrationCleanupThread extends Daemon {
+
+ @Override
+ public void run() {
+ setName("Migration Cleanup Thread");
+ while (stillMaster()) {
+ if (!migrations.isEmpty()) {
+ try {
+ cleanupOfflineMigrations();
+ cleanupNonexistentMigrations(getConnector());
+ } catch (Exception ex) {
+ log.error("Error cleaning up migrations", ex);
+ }
+ }
+ UtilWaitThread.sleep(TIME_BETWEEN_MIGRATION_CLEANUPS);
+ }
+ }
+
+ /**
+ * If a migrating tablet splits, and the tablet dies before sending the master a message, the migration will refer to a non-existing tablet, so it can never
+ * complete. Periodically scan the metadata table and remove any migrating tablets that no longer exist.
+ */
+ private void cleanupNonexistentMigrations(final Connector connector) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ Set<KeyExtent> found = new HashSet<KeyExtent>();
+ for (Entry<Key,Value> entry : scanner) {
+ KeyExtent extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
+ if (migrations.containsKey(extent)) {
+ found.add(extent);
+ }
+ }
+ migrations.keySet().retainAll(found);
+ }
+
+ /**
+ * If migrating a tablet for a table that is offline, the migration can never succeed because no tablet server will load the tablet. check for offline
+ * tables and remove their migrations.
+ */
+ private void cleanupOfflineMigrations() {
+ TableManager manager = TableManager.getInstance();
+ for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
+ TableState state = manager.getTableState(tableId);
+ if (TableState.OFFLINE == state) {
+ clearMigrations(tableId);
+ }
+ }
+ }
+ }
+
+ private class StatusThread extends Daemon {
+
+ @Override
+ public void run() {
+ setName("Status Thread");
+ EventCoordinator.Listener eventListener = nextEvent.getListener();
+ while (stillMaster()) {
+ long wait = DEFAULT_WAIT_FOR_WATCHER;
+ try {
+ switch (getMasterGoalState()) {
+ case NORMAL:
+ setMasterState(MasterState.NORMAL);
+ break;
+ case SAFE_MODE:
+ if (getMasterState() == MasterState.NORMAL) {
+ setMasterState(MasterState.SAFE_MODE);
+ }
+ if (getMasterState() == MasterState.HAVE_LOCK) {
+ setMasterState(MasterState.SAFE_MODE);
+ }
+ break;
+ case CLEAN_STOP:
+ switch (getMasterState()) {
+ case NORMAL:
+ setMasterState(MasterState.SAFE_MODE);
+ break;
+ case SAFE_MODE: {
+ int count = nonMetaDataTabletsAssignedOrHosted();
+ log.debug(String.format("There are %d non-metadata tablets assigned or hosted", count));
+ if (count == 0)
+ setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
+ }
+ break;
+ case UNLOAD_METADATA_TABLETS: {
+ int count = assignedOrHosted(METADATA_TABLE_ID);
+ log.debug(String.format("There are %d metadata tablets assigned or hosted", count));
+ if (count == 0)
+ setMasterState(MasterState.UNLOAD_ROOT_TABLET);
+ }
+ break;
+ case UNLOAD_ROOT_TABLET: {
+ int count = assignedOrHosted(METADATA_TABLE_ID);
+ if (count > 0) {
+ log.debug(String.format("%d metadata tablets online", count));
+ setMasterState(MasterState.UNLOAD_ROOT_TABLET);
+ }
+ int root_count = assignedOrHosted(ROOT_TABLE_ID);
+ if (root_count > 0)
+ log.debug("The root tablet is still assigned or hosted");
+ if (count + root_count == 0) {
+ Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
+ log.debug("stopping " + currentServers.size() + " tablet servers");
+ for (TServerInstance server : currentServers) {
+ try {
+ serversToShutdown.add(server);
+ tserverSet.getConnection(server).fastHalt(masterLock);
+ } catch (TException e) {
+ // its probably down, and we don't care
+ } finally {
+ tserverSet.remove(server);
+ }
+ }
+ if (currentServers.size() == 0)
+ setMasterState(MasterState.STOP);
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ wait = updateStatus();
+ eventListener.waitForEvents(wait);
+ } catch (Throwable t) {
+ log.error("Error balancing tablets", t);
+ UtilWaitThread.sleep(WAIT_BETWEEN_ERRORS);
+ }
+ }
+ }
+
+ private long updateStatus() throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ tserverStatus = Collections.synchronizedSortedMap(gatherTableInformation());
+ checkForHeldServer(tserverStatus);
+
+ if (!badServers.isEmpty()) {
+ log.debug("not balancing because the balance information is out-of-date " + badServers.keySet());
+ } else if (notHosted() > 0) {
+ log.debug("not balancing because there are unhosted tablets: " + notHosted());
+ } else if (getMasterGoalState() == MasterGoalState.CLEAN_STOP) {
+ log.debug("not balancing because the master is attempting to stop cleanly");
+ } else if (!serversToShutdown.isEmpty()) {
+ log.debug("not balancing while shutting down servers " + serversToShutdown);
+ } else {
+ return balanceTablets();
+ }
+ return DEFAULT_WAIT_FOR_WATCHER;
+ }
+
+ private void checkForHeldServer(SortedMap<TServerInstance,TabletServerStatus> tserverStatus) {
+ TServerInstance instance = null;
+ int crazyHoldTime = 0;
+ int someHoldTime = 0;
+ final long maxWait = getSystemConfiguration().getTimeInMillis(Property.TSERV_HOLD_TIME_SUICIDE);
+ for (Entry<TServerInstance,TabletServerStatus> entry : tserverStatus.entrySet()) {
+ if (entry.getValue().getHoldTime() > 0) {
+ someHoldTime++;
+ if (entry.getValue().getHoldTime() > maxWait) {
+ instance = entry.getKey();
+ crazyHoldTime++;
+ }
+ }
+ }
+ if (crazyHoldTime == 1 && someHoldTime == 1 && tserverStatus.size() > 1) {
+ log.warn("Tablet server " + instance + " exceeded maximum hold time: attempting to kill it");
+ try {
+ TServerConnection connection = tserverSet.getConnection(instance);
+ if (connection != null)
+ connection.fastHalt(masterLock);
+ } catch (TException e) {
+ log.error(e, e);
+ }
+ tserverSet.remove(instance);
+ }
+ }
+
+ private long balanceTablets() {
+ List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
+ Set<KeyExtent> migrationsCopy = new HashSet<KeyExtent>();
+ synchronized (migrations) {
+ migrationsCopy.addAll(migrations.keySet());
+ }
+ long wait = tabletBalancer.balance(Collections.unmodifiableSortedMap(tserverStatus), Collections.unmodifiableSet(migrationsCopy), migrationsOut);
+
+ for (TabletMigration m : TabletBalancer.checkMigrationSanity(tserverStatus.keySet(), migrationsOut)) {
+ if (migrations.containsKey(m.tablet)) {
+ log.warn("balancer requested migration more than once, skipping " + m);
+ continue;
+ }
+ migrations.put(m.tablet, m.newServer);
+ log.debug("migration " + m);
+ }
+ if (migrationsOut.size() > 0) {
+ nextEvent.event("Migrating %d more tablets, %d total", migrationsOut.size(), migrations.size());
+ }
+ return wait;
+ }
+
+ }
+
+ private SortedMap<TServerInstance,TabletServerStatus> gatherTableInformation() {
+ long start = System.currentTimeMillis();
+ SortedMap<TServerInstance,TabletServerStatus> result = new TreeMap<TServerInstance,TabletServerStatus>();
+ Set<TServerInstance> currentServers = tserverSet.getCurrentServers();
+ for (TServerInstance server : currentServers) {
+ try {
+ Thread t = Thread.currentThread();
+ String oldName = t.getName();
+ try {
+ t.setName("Getting status from " + server);
+ TServerConnection connection = tserverSet.getConnection(server);
+ if (connection == null)
+ throw new IOException("No connection to " + server);
+ TabletServerStatus status = connection.getTableMap(false);
+ result.put(server, status);
+ } finally {
+ t.setName(oldName);
+ }
+ } catch (Exception ex) {
+ log.error("unable to get tablet server status " + server + " " + ex.toString());
+ log.debug("unable to get tablet server status " + server, ex);
+ if (badServers.get(server).incrementAndGet() > MAX_BAD_STATUS_COUNT) {
+ log.warn("attempting to stop " + server);
+ try {
+ TServerConnection connection = tserverSet.getConnection(server);
+ if (connection != null)
+ connection.halt(masterLock);
+ } catch (TTransportException e) {
+ // ignore: it's probably down
+ } catch (Exception e) {
+ log.info("error talking to troublesome tablet server ", e);
+ }
+ badServers.remove(server);
+ tserverSet.remove(server);
+ }
+ }
+ }
+ synchronized (badServers) {
+ badServers.keySet().retainAll(currentServers);
+ badServers.keySet().removeAll(result.keySet());
+ }
+ log.debug(String.format("Finished gathering information from %d servers in %.2f seconds", result.size(), (System.currentTimeMillis() - start) / 1000.));
+ return result;
+ }
+
+ public void run() throws IOException, InterruptedException, KeeperException {
+ final String zroot = ZooUtil.getRoot(instance);
+
+ getMasterLock(zroot + Constants.ZMASTER_LOCK);
+
+ recoveryManager = new RecoveryManager(this);
+
+ TableManager.getInstance().addObserver(this);
+
+ StatusThread statusThread = new StatusThread();
+ statusThread.start();
+
+ MigrationCleanupThread migrationCleanupThread = new MigrationCleanupThread();
+ migrationCleanupThread.start();
+
+ tserverSet.startListeningForTabletServerChanges();
+
+ ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ nextEvent.event("Noticed recovery changes", event.getType());
+ try {
+ // watcher only fires once, add it back
+ ZooReaderWriter.getInstance().getChildren(zroot + Constants.ZRECOVERY, this);
+ } catch (Exception e) {
+ log.error("Failed to add log recovery watcher back", e);
+ }
+ }
+ });
+
+ Credentials systemCreds = SystemCredentials.get();
+ watchers.add(new TabletGroupWatcher(this, new MetaDataStateStore(instance, systemCreds, this), null));
+ watchers.add(new TabletGroupWatcher(this, new RootTabletStateStore(instance, systemCreds, this), watchers.get(0)));
+ watchers.add(new TabletGroupWatcher(this, new ZooTabletStateStore(new ZooStore(zroot)), watchers.get(1)));
+ for (TabletGroupWatcher watcher : watchers) {
+ watcher.start();
+ }
+
+ // Once we are sure the upgrade is complete, we can safely allow fate use.
+ waitForMetadataUpgrade.await();
+
+ try {
+ final AgeOffStore<Master> store = new AgeOffStore<Master>(new org.apache.accumulo.fate.ZooStore<Master>(ZooUtil.getRoot(instance) + Constants.ZFATE,
+ ZooReaderWriter.getInstance()), 1000 * 60 * 60 * 8);
+
+ int threads = this.getConfiguration().getConfiguration().getCount(Property.MASTER_FATE_THREADPOOL_SIZE);
+
+ fate = new Fate<Master>(this, store, threads);
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+
+ @Override
+ public void run() {
+ store.ageOff();
+ }
+ }, 63000, 63000);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+
+ Processor<Iface> processor = new Processor<Iface>(RpcWrapper.service(new MasterClientServiceHandler(this)));
+ ServerAddress sa = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_CLIENTPORT, processor, "Master",
+ "Master Client Service Handler", null, Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
+ clientService = sa.server;
+ String address = sa.address.toString();
+ log.info("Setting master lock data to " + address);
+ masterLock.replaceLockData(address.getBytes());
+
+ while (!clientService.isServing()) {
+ UtilWaitThread.sleep(100);
+ }
+ while (clientService.isServing()) {
+ UtilWaitThread.sleep(500);
+ }
+ log.info("Shutting down fate.");
+ fate.shutdown();
+
+ final long deadline = System.currentTimeMillis() + MAX_CLEANUP_WAIT_TIME;
+ statusThread.join(remaining(deadline));
+
+ // quit, even if the tablet servers somehow jam up and the watchers
+ // don't stop
+ for (TabletGroupWatcher watcher : watchers) {
+ watcher.join(remaining(deadline));
+ }
+ log.info("exiting");
+ }
+
+ private long remaining(long deadline) {
+ return Math.max(1, deadline - System.currentTimeMillis());
+ }
+
+ public ZooLock getMasterLock() {
+ return masterLock;
+ }
+
+ private static class MasterLockWatcher implements ZooLock.AsyncLockWatcher {
+
+ boolean acquiredLock = false;
+ boolean failedToAcquireLock = false;
+
+ @Override
+ public void lostLock(LockLossReason reason) {
+ Halt.halt("Master lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
+ }
+
+ @Override
+ public void unableToMonitorLockNode(final Throwable e) {
+ Halt.halt(-1, new Runnable() {
+ @Override
+ public void run() {
+ log.fatal("No longer able to monitor master lock node", e);
+ }
+ });
+
+ }
+
+ @Override
+ public synchronized void acquiredLock() {
+ log.debug("Acquired master lock");
+
+ if (acquiredLock || failedToAcquireLock) {
+ Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ acquiredLock = true;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void failedToAcquireLock(Exception e) {
+ log.warn("Failed to get master lock " + e);
+
+ if (acquiredLock) {
+ Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ failedToAcquireLock = true;
+ notifyAll();
+ }
+
+ public synchronized void waitForChange() {
+ while (!acquiredLock && !failedToAcquireLock) {
+ try {
+ wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+
+ private void getMasterLock(final String zMasterLoc) throws KeeperException, InterruptedException {
+ log.info("trying to get master lock");
+
+ final String masterClientAddress = hostname + ":" + getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
+
+ while (true) {
+
+ MasterLockWatcher masterLockWatcher = new MasterLockWatcher();
+ masterLock = new ZooLock(zMasterLoc);
+ masterLock.lockAsync(masterLockWatcher, masterClientAddress.getBytes());
+
+ masterLockWatcher.waitForChange();
+
+ if (masterLockWatcher.acquiredLock) {
+ break;
+ }
+
+ if (!masterLockWatcher.failedToAcquireLock) {
+ throw new IllegalStateException("master lock in unknown state");
+ }
+
+ masterLock.tryToCancelAsyncLockOrUnlock();
+
+ UtilWaitThread.sleep(TIME_TO_WAIT_BETWEEN_LOCK_CHECKS);
+ }
+
+ setMasterState(MasterState.HAVE_LOCK);
+ }
+
+ public static void main(String[] args) throws Exception {
+ try {
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+
+ ServerOpts opts = new ServerOpts();
+ final String app = "master";
+ opts.parseArgs(app, args);
+ String hostname = opts.getAddress();
+ Accumulo.setupLogging(app);
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ VolumeManager fs = VolumeManagerImpl.get();
+ Accumulo.init(fs, conf, app);
+ Master master = new Master(conf, fs, hostname);
+ Accumulo.enableTracing(hostname, app);
+ master.run();
+ } catch (Exception ex) {
+ log.error("Unexpected exception, exiting", ex);
+ System.exit(1);
+ }
+ }
+
+ @Override
+ public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ DeadServerList obit = new DeadServerList(ZooUtil.getRoot(instance) + Constants.ZDEADTSERVERS);
+ if (added.size() > 0) {
+ log.info("New servers: " + added);
+ for (TServerInstance up : added)
+ obit.delete(up.hostPort());
+ }
+ for (TServerInstance dead : deleted) {
+ String cause = "unexpected failure";
+ if (serversToShutdown.contains(dead))
+ cause = "clean shutdown"; // maybe an incorrect assumption
+ if (!getMasterGoalState().equals(MasterGoalState.CLEAN_STOP))
+ obit.post(dead.hostPort(), cause);
+ }
+
+ Set<TServerInstance> unexpected = new HashSet<TServerInstance>(deleted);
+ unexpected.removeAll(this.serversToShutdown);
+ if (unexpected.size() > 0) {
+ if (stillMaster() && !getMasterGoalState().equals(MasterGoalState.CLEAN_STOP)) {
+ log.warn("Lost servers " + unexpected);
+ }
+ }
+ serversToShutdown.removeAll(deleted);
+ badServers.keySet().removeAll(deleted);
+ // clear out any bad server with the same host/port as a new server
+ synchronized (badServers) {
+ cleanListByHostAndPort(badServers.keySet(), deleted, added);
+ }
+ synchronized (serversToShutdown) {
+ cleanListByHostAndPort(serversToShutdown, deleted, added);
+ }
+
+ synchronized (migrations) {
+ Iterator<Entry<KeyExtent,TServerInstance>> iter = migrations.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<KeyExtent,TServerInstance> entry = iter.next();
+ if (deleted.contains(entry.getValue())) {
+ log.info("Canceling migration of " + entry.getKey() + " to " + entry.getValue());
+ iter.remove();
+ }
+ }
+ }
+ nextEvent.event("There are now %d tablet servers", current.size());
+ }
+
+ private static void cleanListByHostAndPort(Collection<TServerInstance> badServers, Set<TServerInstance> deleted, Set<TServerInstance> added) {
+ Iterator<TServerInstance> badIter = badServers.iterator();
+ while (badIter.hasNext()) {
+ TServerInstance bad = badIter.next();
+ for (TServerInstance add : added) {
+ if (bad.hostPort().equals(add.hostPort())) {
+ badIter.remove();
+ break;
+ }
+ }
+ for (TServerInstance del : deleted) {
+ if (bad.hostPort().equals(del.hostPort())) {
+ badIter.remove();
+ break;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void stateChanged(String tableId, TableState state) {
+ nextEvent.event("Table state in zookeeper changed for %s to %s", tableId, state);
+ if (TableState.OFFLINE == state) {
+ clearMigrations(tableId);
+ }
+ }
+
+ @Override
+ public void initialize(Map<String,TableState> tableIdToStateMap) {}
+
+ @Override
+ public void sessionExpired() {}
+
+ @Override
+ public Set<String> onlineTables() {
+ Set<String> result = new HashSet<String>();
+ if (getMasterState() != MasterState.NORMAL) {
+ if (getMasterState() != MasterState.UNLOAD_METADATA_TABLETS)
+ result.add(MetadataTable.ID);
+ if (getMasterState() != MasterState.UNLOAD_ROOT_TABLET)
+ result.add(RootTable.ID);
+ return result;
+ }
+ TableManager manager = TableManager.getInstance();
+
+ for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
+ TableState state = manager.getTableState(tableId);
+ if (state != null) {
+ if (state == TableState.ONLINE)
+ result.add(tableId);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public Set<TServerInstance> onlineTabletServers() {
+ return tserverSet.getCurrentServers();
+ }
+
+ @Override
+ public Collection<MergeInfo> merges() {
+ List<MergeInfo> result = new ArrayList<MergeInfo>();
+ for (String tableId : Tables.getIdToNameMap(instance).keySet()) {
+ result.add(getMergeInfo(new Text(tableId)));
+ }
+ return result;
+ }
+
+ // recovers state from the persistent transaction to shutdown a server
+ public void shutdownTServer(TServerInstance server) {
+ nextEvent.event("Tablet Server shutdown requested for %s", server);
+ serversToShutdown.add(server);
+ }
+
+ public EventCoordinator getEventCoordinator() {
+ return nextEvent;
+ }
+
+ public Instance getInstance() {
+ return this.instance;
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ public ServerConfiguration getConfiguration() {
+ return serverConfig;
+ }
+
+ public VolumeManager getFileSystem() {
+ return this.fs;
+ }
+
+ public void assignedTablet(KeyExtent extent) {
+ if (extent.isMeta()) {
+ if (getMasterState().equals(MasterState.UNLOAD_ROOT_TABLET)) {
+ setMasterState(MasterState.UNLOAD_METADATA_TABLETS);
+ }
+ }
+ if (extent.isRootTablet()) {
+ // probably too late, but try anyhow
+ if (getMasterState().equals(MasterState.STOP)) {
+ setMasterState(MasterState.UNLOAD_ROOT_TABLET);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index 0c2e1f0,0000000..5e478cb
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@@ -1,190 -1,0 +1,192 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.recovery;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.log.SortedLogState;
+import org.apache.accumulo.server.master.recovery.HadoopLogCloser;
+import org.apache.accumulo.server.master.recovery.LogCloser;
+import org.apache.accumulo.server.master.recovery.RecoveryPath;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class RecoveryManager {
+
+ private static Logger log = Logger.getLogger(RecoveryManager.class);
+
+ private Map<String,Long> recoveryDelay = new HashMap<String,Long>();
+ private Set<String> closeTasksQueued = new HashSet<String>();
+ private Set<String> sortsQueued = new HashSet<String>();
+ private ScheduledExecutorService executor;
+ private Master master;
+ private ZooCache zooCache;
+
+ public RecoveryManager(Master master) {
+ this.master = master;
+ executor = Executors.newScheduledThreadPool(4, new NamingThreadFactory("Walog sort starter "));
+ zooCache = new ZooCache();
+ try {
+ List<String> workIDs = new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).getWorkQueued();
+ sortsQueued.addAll(workIDs);
+ } catch (Exception e) {
+ log.warn(e, e);
+ }
+ }
+
+ private class LogSortTask implements Runnable {
+ private String source;
+ private String destination;
+ private String sortId;
+ private LogCloser closer;
+
+ public LogSortTask(LogCloser closer, String source, String destination, String sortId) {
+ this.closer = closer;
+ this.source = source;
+ this.destination = destination;
+ this.sortId = sortId;
+ }
+
+ @Override
+ public void run() {
+ boolean rescheduled = false;
+ try {
+
+ long time = closer.close(master.getConfiguration().getConfiguration(), master.getFileSystem(), new Path(source));
+
+ if (time > 0) {
+ executor.schedule(this, time, TimeUnit.MILLISECONDS);
+ rescheduled = true;
+ } else {
+ initiateSort(sortId, source, destination);
+ }
+ } catch (FileNotFoundException e) {
+ log.debug("Unable to initate log sort for " + source + ": " + e);
+ } catch (Exception e) {
+ log.warn("Failed to initiate log sort " + source, e);
+ } finally {
+ if (!rescheduled) {
+ synchronized (RecoveryManager.this) {
+ closeTasksQueued.remove(sortId);
+ }
+ }
+ }
+ }
+
+ }
+
+ private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
+ String work = source + "|" + destination;
- new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(Constants.UTF8));
++ new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(UTF_8));
+
+ synchronized (this) {
+ sortsQueued.add(sortId);
+ }
+
+ final String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId;
+ log.info("Created zookeeper entry " + path + " with data " + work);
+ }
+
+ public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>> walogs) throws IOException {
+ boolean recoveryNeeded = false;
+
+ for (Collection<String> logs : walogs) {
+ for (String walog : logs) {
+
+ String switchedWalog = VolumeUtil.switchVolume(walog, FileType.WAL, ServerConstants.getVolumeReplacements());
+ if (switchedWalog != null) {
+ // replaces the volume used for sorting, but do not change entry in metadata table. When the tablet loads it will change the metadata table entry. If
+ // the tablet has the same replacement config, then it will find the sorted log.
+ log.info("Volume replaced " + walog + " -> " + switchedWalog);
+ walog = switchedWalog;
+ }
+
+ String parts[] = walog.split("/");
+ String sortId = parts[parts.length - 1];
+ String filename = master.getFileSystem().getFullPath(FileType.WAL, walog).toString();
+ String dest = RecoveryPath.getRecoveryPath(master.getFileSystem(), new Path(filename)).toString();
+ log.debug("Recovering " + filename + " to " + dest);
+
+ boolean sortQueued;
+ synchronized (this) {
+ sortQueued = sortsQueued.contains(sortId);
+ }
+
+ if (sortQueued && zooCache.get(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY + "/" + sortId) == null) {
+ synchronized (this) {
+ sortsQueued.remove(sortId);
+ }
+ }
+
+ if (master.getFileSystem().exists(SortedLogState.getFinishedMarkerPath(dest))) {
+ synchronized (this) {
+ closeTasksQueued.remove(sortId);
+ recoveryDelay.remove(sortId);
+ sortsQueued.remove(sortId);
+ }
+ continue;
+ }
+
+ recoveryNeeded = true;
+ synchronized (this) {
+ if (!closeTasksQueued.contains(sortId) && !sortsQueued.contains(sortId)) {
+ AccumuloConfiguration aconf = master.getConfiguration().getConfiguration();
+ LogCloser closer = aconf.instantiateClassProperty(Property.MASTER_WALOG_CLOSER_IMPLEMETATION, LogCloser.class, new HadoopLogCloser());
+ Long delay = recoveryDelay.get(sortId);
+ if (delay == null) {
+ delay = master.getSystemConfiguration().getTimeInMillis(Property.MASTER_RECOVERY_DELAY);
+ } else {
+ delay = Math.min(2 * delay, 1000 * 60 * 5l);
+ }
+
+ log.info("Starting recovery of " + filename + " (in : " + (delay / 1000) + "s), tablet " + extent + " holds a reference");
+
+ executor.schedule(new LogSortTask(closer, filename, dest, sortId), delay, TimeUnit.MILLISECONDS);
+ closeTasksQueued.add(sortId);
+ recoveryDelay.put(sortId, delay);
+ }
+ }
+ }
+ }
+ return recoveryNeeded;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
index f981bae,0000000..a674557
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
@@@ -1,49 -1,0 +1,51 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.state;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+public class SetGoalState {
+
+ /**
+ * Utility program that will change the goal state for the master from the command line.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length != 1 || MasterGoalState.valueOf(args[0]) == null) {
+ System.err.println("Usage: accumulo " + SetGoalState.class.getName() + " [NORMAL|SAFE_MODE|CLEAN_STOP]");
+ System.exit(-1);
+ }
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ Accumulo.waitForZookeeperAndHdfs(fs);
- ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(Constants.UTF8),
++ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, args[0].getBytes(UTF_8),
+ NodeExistsPolicy.OVERWRITE);
+ }
+
+}
[19/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index cf1f0ea,0000000..da83386
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,407 -1,0 +1,409 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.6.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN,
+ }
+
+ public static enum TokenSource {
+ FILE, INLINE;
+
+ private String prefix;
+
+ private TokenSource() {
+ prefix = name().toLowerCase() + ":";
+ }
+
+ public String prefix() {
+ return prefix;
+ }
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.6.0
+ */
+ public static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.6.0
+ */
+ public static enum GeneralOpts {
+ LOG_LEVEL,
+ VISIBILITY_CACHE_SIZE
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.6.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, token);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
+ TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
+ * Configuration. Token file created with "bin/accumulo create-token".
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param tokenFile
+ * the path to the token file in DFS
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, tokenFile);
+
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
+ }
+
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ * @see #setConnectorInfo(Class, Configuration, String, String)
+ */
+ public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
+ String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
+ if (token == null || token.isEmpty())
+ return null;
+ if (token.startsWith(TokenSource.INLINE.prefix())) {
+ String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
+ if (args.length == 2)
- return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(Constants.UTF8)));
++ return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(UTF_8)));
+ } else if (token.startsWith(TokenSource.FILE.prefix())) {
+ String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
+ return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
+ }
+
+ throw new IllegalStateException("Token was not properly serialized into the configuration");
+ }
+
+ /**
+ * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
+ *
+ * @param conf
+ * the Hadoop context for the configured job
+ * @return path to the token file as a String
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
+ FSDataInputStream in = null;
+ try {
+ URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
+ Path path = null;
+ for (URI u : uris) {
+ if (u.toString().equals(tokenFile)) {
+ path = new Path(u);
+ }
+ }
+ if (path == null) {
+ throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
+ }
+ FileSystem fs = FileSystem.get(conf);
+ in = fs.open(path);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
+ }
+ java.util.Scanner fileScanner = new java.util.Scanner(in);
+ try {
+ while (fileScanner.hasNextLine()) {
+ Credentials creds = Credentials.deserialize(fileScanner.nextLine());
+ if (principal.equals(creds.getPrincipal())) {
+ return creds.getToken();
+ }
+ }
+ throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
+ } finally {
+ if (fileScanner != null && fileScanner.ioException() == null)
+ fileScanner.close();
+ else if (fileScanner.ioException() != null)
+ throw new RuntimeException(fileScanner.ioException());
+ }
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+ if (clientConfig != null) {
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
+ }
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.6.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ ArgumentChecker.notNull(instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.6.0
+ * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+ if (clientConfigString == null) {
+ String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+ String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
+ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
+ } else {
+ return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
+ }
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.6.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ ArgumentChecker.notNull(level);
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.6.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
index 1ed23e3,0000000..f8a5d03
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@@ -1,795 -1,0 +1,795 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.core.util.ArgumentChecker.notNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
+import org.apache.accumulo.core.client.mock.impl.MockTabletLocator;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @since 1.6.0
+ */
+public class InputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Scanner}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ScanOpts {
+ TABLE_NAME, AUTHORIZATIONS, RANGES, COLUMNS, ITERATORS, TABLE_CONFIGS
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ AUTO_ADJUST_RANGES, SCAN_ISOLATION, USE_LOCAL_ITERATORS, SCAN_OFFLINE
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
+ notNull(tableName);
+ conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @since 1.6.0
+ */
+ public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param auths
+ * the user's authorizations
+ * @since 1.6.0
+ */
+ public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf, Authorizations auths) {
+ if (auths != null && !auths.isEmpty())
+ conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the Accumulo scan authorizations
+ * @since 1.6.0
+ * @see #setScanAuthorizations(Class, Configuration, Authorizations)
+ */
+ public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
+ String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
- return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(Constants.UTF8));
++ return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(UTF_8));
+ }
+
+ /**
+ * Sets the input ranges to scan on all input tables for this job. If not set, the entire table will be scanned.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param ranges
+ * the ranges that will be mapped over
+ * @throws IllegalArgumentException
+ * if the ranges cannot be encoded into base 64
+ * @since 1.6.0
+ */
+ public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
+ notNull(ranges);
+
+ ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
+ try {
+ for (Range r : ranges) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ r.write(new DataOutputStream(baos));
+ rangeStrings.add(Base64.encodeBase64String(baos.toByteArray()));
+ }
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES), rangeStrings.toArray(new String[0]));
+ } catch (IOException ex) {
+ throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
+ }
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.6.0
+ * @see #setRanges(Class, Configuration, Collection)
+ */
+ public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
+
+ Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
+ List<Range> ranges = new ArrayList<Range>();
+ for (String rangeString : encodedRanges) {
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes()));
+ Range range = new Range();
+ range.readFields(new DataInputStream(bais));
+ ranges.add(range);
+ }
+ return ranges;
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a list of iterators
+ * @since 1.6.0
+ * @see #addIterator(Class, Configuration, IteratorSetting)
+ */
+ public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
+ String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
+
+ // If no iterators are present, return an empty list
+ if (iterators == null || iterators.isEmpty())
+ return new ArrayList<IteratorSetting>();
+
+ // Compose the set of iterators encoded in the job configuration
+ StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
+ List<IteratorSetting> list = new ArrayList<IteratorSetting>();
+ try {
+ while (tokens.hasMoreTokens()) {
+ String itstring = tokens.nextToken();
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
+ list.add(new IteratorSetting(new DataInputStream(bais)));
+ bais.close();
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("couldn't decode iterator settings");
+ }
+ return list;
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for the single input table on this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @throws IllegalArgumentException
+ * if the column family is null
+ * @since 1.6.0
+ */
+ public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ notNull(columnFamilyColumnQualifierPairs);
+ String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
+ }
+
+ public static String[] serializeColumns(Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ notNull(columnFamilyColumnQualifierPairs);
+ ArrayList<String> columnStrings = new ArrayList<String>(columnFamilyColumnQualifierPairs.size());
+ for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
+
+ if (column.getFirst() == null)
+ throw new IllegalArgumentException("Column family can not be null");
+
+ String col = Base64.encodeBase64String(TextUtil.getBytes(column.getFirst()));
+ if (column.getSecond() != null)
+ col += ":" + Base64.encodeBase64String(TextUtil.getBytes(column.getSecond()));
+ columnStrings.add(col);
+ }
+
+ return columnStrings.toArray(new String[0]);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a set of columns
+ * @since 1.6.0
+ * @see #fetchColumns(Class, Configuration, Collection)
+ */
+ public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass, Configuration conf) {
+ notNull(conf);
+ String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
+ List<String> serialized = new ArrayList<String>();
+ if (confValue != null) {
+ // Split and include any trailing empty strings to allow empty column families
+ for (String val : confValue.split(",", -1)) {
+ serialized.add(val);
+ }
+ }
+ return deserializeFetchedColumns(serialized);
+ }
+
+ public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
+ Set<Pair<Text,Text>> columns = new HashSet<Pair<Text,Text>>();
+
+ if (null == serialized) {
+ return columns;
+ }
+
+ for (String col : serialized) {
+ int idx = col.indexOf(":");
- Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8)));
- Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8)));
++ Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(UTF_8)));
++ Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(UTF_8)));
+ columns.add(new Pair<Text,Text>(cf, cq));
+ }
+ return columns;
+ }
+
+ /**
+ * Encode an iterator on the input for the single input table associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param cfg
+ * the configuration of the iterator
+ * @throws IllegalArgumentException
+ * if the iterator can't be serialized into the configuration
+ * @since 1.6.0
+ */
+ public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String newIter;
+ try {
+ cfg.write(new DataOutputStream(baos));
+ newIter = Base64.encodeBase64String(baos.toByteArray());
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize IteratorSetting");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
+ String iterators = conf.get(confKey);
+ // No iterators specified yet, create a new string
+ if (iterators == null || iterators.isEmpty()) {
+ iterators = newIter;
+ } else {
+ // append the next iterator & reset
+ iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
+ }
+ // Store the iterators w/ the job
+ conf.set(confKey, iterators);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Class, Configuration, Collection)
+ * @since 1.6.0
+ */
+ public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(Class, Configuration, boolean)
+ */
+ public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setScanIsolation(Class, Configuration, boolean)
+ */
+ public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setLocalIterators(Class, Configuration, boolean)
+ */
+ public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineTableScan(Class, Configuration, boolean)
+ */
+ public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
+ }
+
+ /**
+ * Sets configurations for multiple tables at a time.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param configs
+ * an array of {@link InputTableConfig} objects to associate with the job
+ * @since 1.6.0
+ */
+ public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String,InputTableConfig> configs) {
+ MapWritable mapWritable = new MapWritable();
+ for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
+ mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ mapWritable.write(new DataOutputStream(baos));
+ } catch (IOException e) {
+ throw new IllegalStateException("Table configuration could not be serialized.");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
+ conf.set(confKey, Base64.encodeBase64String(baos.toByteArray()));
+ }
+
+ /**
+ * Returns all {@link InputTableConfig} objects associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return all of the table query configs for the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
+ Map<String,InputTableConfig> configs = new HashMap<String,InputTableConfig>();
+ Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
+ if (defaultConfig != null)
+ configs.put(defaultConfig.getKey(), defaultConfig.getValue());
+ String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
+ MapWritable mapWritable = new MapWritable();
+ if (configString != null) {
+ try {
+ byte[] bytes = Base64.decodeBase64(configString.getBytes());
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ mapWritable.readFields(new DataInputStream(bais));
+ bais.close();
+ } catch (IOException e) {
+ throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
+ }
+ }
+ for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
+ configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());
+
+ return configs;
+ }
+
+ /**
+ * Returns the {@link InputTableConfig} for the given table
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table name for which to fetch the table query config
+ * @return the table query config for the given table name (if it exists) and null if it does not
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf, String tableName) {
+ Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
+ return queryConfigs.get(tableName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableId
+ * The table id for which to initialize the {@link TabletLocator}
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf, String tableId) throws TableNotFoundException {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
+ if ("MockInstance".equals(instanceType))
+ return new MockTabletLocator();
+ Instance instance = getInstance(implementingClass, conf);
+ return TabletLocator.getLocator(instance, new Text(tableId));
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.6.0
+ */
+ public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
+
+ Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
+ if (!isConnectorInfoSet(implementingClass, conf))
+ throw new IOException("Input info has not been set.");
+ String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
+ if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
+ throw new IOException("Instance info has not been set.");
+ // validate that we can connect as configured
+ try {
+ String principal = getPrincipal(implementingClass, conf);
+ AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
+ Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
+ if (!c.securityOperations().authenticateUser(principal, token))
+ throw new IOException("Unable to authenticate user");
+
+ if (getInputTableConfigs(implementingClass, conf).size() == 0)
+ throw new IOException("No table set.");
+
+ for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
+ if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
+ throw new IOException("Unable to access table");
+ }
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+ if (!tableConfig.shouldUseLocalIterators()) {
+ if (tableConfig.getIterators() != null) {
+ for (IteratorSetting iter : tableConfig.getIterators()) {
+ if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
+ throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
+ }
+ }
+ }
+ }
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the configuration based on the properties set using the single-table
+ * input methods.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop instance for which to retrieve the configuration
+ * @return the config object built from the single input table properties set on the job
+ * @since 1.6.0
+ */
+ protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(Class<?> implementingClass, Configuration conf) {
+ String tableName = getInputTableName(implementingClass, conf);
+ if (tableName != null) {
+ InputTableConfig queryConfig = new InputTableConfig();
+ List<IteratorSetting> itrs = getIterators(implementingClass, conf);
+ if (itrs != null)
+ queryConfig.setIterators(itrs);
+ Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
+ if (columns != null)
+ queryConfig.fetchColumns(columns);
+ List<Range> ranges = null;
+ try {
+ ranges = getRanges(implementingClass, conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (ranges != null)
+ queryConfig.setRanges(ranges);
+
+ queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)).setUseIsolatedScanners(isIsolated(implementingClass, conf))
+ .setUseLocalIterators(usesLocalIterators(implementingClass, conf)).setOfflineScan(isOfflineScan(implementingClass, conf));
+ return Maps.immutableEntry(tableName, queryConfig);
+ }
+ return null;
+ }
+
+ public static Map<String,Map<KeyExtent,List<Range>>> binOffline(String tableId, List<Range> ranges, Instance instance, Connector conn)
+ throws AccumuloException, TableNotFoundException {
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ KeyExtent lastExtent = null;
+ while (rowIter.hasNext()) {
+ Iterator<Map.Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Map.Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily().equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
+ location = entry.getValue().toString();
+ }
+
+ if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+ return binnedRanges;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
index 0ba42cd,0000000..26680dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
@@@ -1,204 -1,0 +1,205 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * @since 1.6.0
+ */
+public class OutputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link BatchWriter}.
+ *
+ * @since 1.6.0
+ */
+ public static enum WriteOpts {
+ DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ CAN_CREATE_TABLES, SIMULATION_MODE
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setDefaultTableName(Class<?> implementingClass, Configuration conf, String tableName) {
+ if (tableName != null)
+ conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the default table name
+ * @since 1.6.0
+ * @see #setDefaultTableName(Class, Configuration, String)
+ */
+ public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.6.0
+ */
+ public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf, BatchWriterConfig bwConfig) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String serialized;
+ try {
+ bwConfig.write(new DataOutputStream(baos));
- serialized = new String(baos.toByteArray(), Constants.UTF8);
++ serialized = new String(baos.toByteArray(), UTF_8);
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the configuration object
+ * @since 1.6.0
+ * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
+ */
+ public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
+ String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
+ BatchWriterConfig bwConfig = new BatchWriterConfig();
+ if (serialized == null || serialized.isEmpty()) {
+ return bwConfig;
+ } else {
+ try {
- ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(Constants.UTF8));
++ ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
+ bwConfig.readFields(new DataInputStream(bais));
+ bais.close();
+ return bwConfig;
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ }
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setCreateTables(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.6.0
+ * @see #setCreateTables(Class, Configuration, boolean)
+ */
+ public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setSimulationMode(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setSimulationMode(Class, Configuration, boolean)
+ */
+ public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 1541fae,0753282..9e26672
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@@ -26,11 -28,10 +28,10 @@@ import java.util.Arrays
import java.util.Scanner;
import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
-import org.apache.commons.codec.binary.Base64;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.DistributedCacheHelper;
+import org.apache.accumulo.core.util.Base64;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
index 0d3aca2,d95a647..95fcf53
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
@@@ -16,20 -16,18 +16,21 @@@
*/
package org.apache.accumulo.core.client.mock;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.io.ByteArrayInputStream;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Writer;
-import jline.ConsoleReader;
+import org.apache.commons.io.output.WriterOutputStream;
-import org.apache.accumulo.core.util.shell.Shell;
import org.apache.commons.cli.CommandLine;
+import jline.console.ConsoleReader;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.core.util.shell.ShellOptionsJC;
/**
* An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes.
@@@ -38,29 -36,12 +39,29 @@@ public class MockShell extends Shell
private static final String NEWLINE = "\n";
protected InputStream in;
- protected Writer writer;
-
- public MockShell(InputStream in, Writer writer) throws IOException {
+ protected OutputStream out;
+
+ /**
+ * Will only be set if you use either the Writer constructor or the setWriter(Writer) method
+ * @deprecated since 1.6.0; use out
+ */
+ @Deprecated
+ protected Writer writer = null;
+
+ public MockShell(InputStream in, OutputStream out) throws IOException {
super();
this.in = in;
- this.writer = writer;
+ this.out = out;
+ // we presume they don't use the writer field unless they use the other constructor.
+ }
+
+ /**
+ * @deprecated since 1.6.0; use OutputStream version
+ */
+ @Deprecated
+ public MockShell(InputStream in, Writer out) throws IOException {
- this(in, new WriterOutputStream(out, Constants.UTF8.name()));
++ this(in, new WriterOutputStream(out, UTF_8.name()));
+ this.writer = out;
}
public boolean config(String... args) {
@@@ -107,7 -79,7 +108,7 @@@
printInfo();
if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
- java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++ java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
try {
while (scanner.hasNextLine() && !hasExited()) {
execCommand(scanner.nextLine(), true, isVerbose());
@@@ -146,20 -118,11 +147,20 @@@
}
/**
- * @param writer
- * the writer to set
+ * @param out
+ * the output stream to set
+ */
+ public void setConsoleWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ /**
+ * @deprecated since 1.6.0; use the OutputStream version
*/
- public void setConsoleWriter(Writer writer) {
- this.writer = writer;
+ @Deprecated
+ public void setConsoleWriter(Writer out) {
- setConsoleWriter(new WriterOutputStream(out, Constants.UTF8.name()));
++ setConsoleWriter(new WriterOutputStream(out, UTF_8.name()));
+ this.writer = out;
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index bd9feba,05aa0c3..4d73979
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@@ -124,14 -123,10 +125,14 @@@ public class PasswordToken implements A
throw new RuntimeException(e);
}
}
-
- private void setPassword(CharBuffer charBuffer) {
+
+ protected void setPassword(byte[] password) {
+ this.password = Arrays.copyOf(password, password.length);
+ }
+
+ protected void setPassword(CharBuffer charBuffer) {
// encode() kicks back a C-string, which is not compatible with the old passwording system
- ByteBuffer bb = Constants.UTF8.encode(charBuffer);
+ ByteBuffer bb = UTF_8.encode(charBuffer);
// create array using byter buffer length
this.password = new byte[bb.remaining()];
bb.get(this.password);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 41a1f3b,0000000..618ff7e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -1,356 -1,0 +1,357 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.TreeMap;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.log4j.Logger;
+
+/**
+ * This class generates documentation to inform users of the available configuration properties in a presentable form.
+ */
+class ConfigurationDocGen {
+ private abstract class Format {
+
+ abstract void beginSection(String section);
+
+ void endSection() {}
+
+ void generate() {
+ pageHeader();
+
+ beginSection("Available Properties");
+ for (Property prefix : prefixes) {
+ if (!prefix.isExperimental()) {
+ prefixSection(prefix);
+ for (Property prop : sortedProps.values()) {
+ if (!prop.isExperimental()) {
+ property(prefix, prop);
+ }
+ }
+ }
+ }
+ endSection();
+
+ beginSection("Property Types");
+ propertyTypeDescriptions();
+ endSection();
+
+ pageFooter();
+ doc.close();
+ }
+
+ abstract String getExt();
+
+ void pageFooter() {}
+
+ // read static header content from resources and output
+ void pageHeader() {
+ appendResource("config-header." + getExt());
+ doc.println();
+ }
+
+ abstract void prefixSection(Property prefix);
+
+ abstract void property(Property prefix, Property prop);
+
+ abstract void propertyTypeDescriptions();
+
+ abstract String sanitize(String str);
+
+ }
+
+ private class HTML extends Format {
+ private boolean highlight;
+
+ private void beginRow() {
+ doc.println(startTag("tr", highlight ? "class='highlight'" : null));
+ highlight = !highlight;
+ }
+
+ @Override
+ void beginSection(String section) {
+ doc.println(t("h1", section, null));
+ highlight = true;
+ doc.println("<table>");
+ }
+
+ private String t(String tag, String cell, String options) {
+ return startTag(tag, options) + cell + "</" + tag + ">";
+ }
+
+ private void cellData(String cell, String options) {
+ doc.println(t("td", cell, options));
+ }
+
+ private void columnNames(String... names) {
+ beginRow();
+ for (String s : names)
+ doc.println(t("th", s, null));
+ endRow();
+ }
+
+ private void endRow() {
+ doc.println("</tr>");
+ }
+
+ @Override
+ void endSection() {
+ doc.println("</table>");
+ }
+
+ @Override
+ String getExt() {
+ return "html";
+ }
+
+ @Override
+ void pageFooter() {
+ doc.println("</body>");
+ doc.println("</html>");
+ }
+
+ @Override
+ void pageHeader() {
+ super.pageHeader();
+ doc.println("<p>Jump to: ");
+ String delimiter = "";
+ for (Property prefix : prefixes) {
+ if (!prefix.isExperimental()) {
+ doc.print(delimiter + "<a href='#" + prefix.name() + "'>" + prefix.getKey() + "*</a>");
+ delimiter = " | ";
+ }
+ }
+ doc.println("</p>");
+ }
+
+ @Override
+ void prefixSection(Property prefix) {
+ beginRow();
+ doc.println(t("td", t("span", prefix.getKey() + '*', "id='" + prefix.name() + "' class='large'"), "colspan='5'"
+ + (prefix.isDeprecated() ? " class='deprecated'" : "")));
+ endRow();
+ beginRow();
+ doc.println(t("td", (prefix.isDeprecated() ? t("b", t("i", "Deprecated. ", null), null) : "") + sanitize(prefix.getDescription()), "colspan='5'"
+ + (prefix.isDeprecated() ? " class='deprecated'" : "")));
+ endRow();
+
+ switch (prefix) {
+ case TABLE_CONSTRAINT_PREFIX:
+ break;
+ case TABLE_ITERATOR_PREFIX:
+ break;
+ case TABLE_LOCALITY_GROUP_PREFIX:
+ break;
+ case TABLE_COMPACTION_STRATEGY_PREFIX:
+ break;
+ default:
+ columnNames("Property", "Type", "ZooKeeper Mutable", "Default Value", "Description");
+ break;
+ }
+ }
+
+ @Override
+ void property(Property prefix, Property prop) {
+ boolean isDeprecated = prefix.isDeprecated() || prop.isDeprecated();
+ if (prop.getKey().startsWith(prefix.getKey())) {
+ beginRow();
+ cellData(prop.getKey(), isDeprecated ? "class='deprecated'" : null);
+ cellData("<b><a href='#" + prop.getType().name() + "'>" + prop.getType().toString().replaceAll(" ", " ") + "</a></b>",
+ isDeprecated ? "class='deprecated'" : null);
+ cellData(isZooKeeperMutable(prop), isDeprecated ? "class='deprecated'" : null);
+ cellData("<pre>" + (prop.getRawDefaultValue().isEmpty() ? " " : sanitize(prop.getRawDefaultValue().replaceAll(" ", " "))) + "</pre>",
+ isDeprecated ? "class='deprecated'" : null);
+ cellData((isDeprecated ? "<b><i>Deprecated.</i></b> " : "") + sanitize(prop.getDescription()), isDeprecated ? "class='deprecated'" : null);
+ endRow();
+ }
+
+ }
+
+ @Override
+ void propertyTypeDescriptions() {
+ columnNames("Property Type", "Description");
+ for (PropertyType type : PropertyType.values()) {
+ if (type == PropertyType.PREFIX)
+ continue;
+ beginRow();
+ cellData("<h3 id='" + type.name() + "'>" + type + "</h3>", null);
+ cellData(type.getFormatDescription(), null);
+ endRow();
+ }
+ }
+
+ @Override
+ String sanitize(String str) {
+ return str.replace("&", "&").replace("<", "<").replace(">", ">").replaceAll("(?:\r\n|\r|\n)", "<br />");
+ }
+
+ private String startTag(String tag, String options) {
+ return "<" + tag + (options != null ? " " + options : "") + ">";
+ }
+
+ }
+
+ private class LaTeX extends Format {
+ @Override
+ void beginSection(String section) {
+ doc.println("\\section{" + section + "}");
+ }
+
+ @Override
+ String getExt() {
+ return "tex";
+ }
+
+ @Override
+ void prefixSection(Property prefix) {
+ boolean depr = prefix.isDeprecated();
+ doc.println("\\subsection{" + prefix.getKey() + "*}" + (depr ? " (Deprecated)" : ""));
+ doc.println((depr ? "\\sout{\\textit{Deprecated.} " : "") + sanitize(prefix.getDescription()) + (depr ? "}" : ""));
+ doc.println();
+ }
+
+ @Override
+ void property(Property prefix, Property prop) {
+ boolean depr = prefix.isDeprecated() || prop.isDeprecated();
+ if (prop.getKey().startsWith(prefix.getKey())) {
+ doc.println("\\subsubsection{" + prop.getKey() + "}");
+ doc.println(strike((depr ? "\\textit{Deprecated.} " : "") + sanitize(prop.getDescription()), depr));
+ doc.println();
+ doc.println(strike("\\textit{Type:} " + prop.getType().name() + "\\\\", depr));
+ doc.println(strike("\\textit{ZooKeeper Mutable:} " + isZooKeeperMutable(prop) + "\\\\", depr));
+ doc.println(strike("\\textit{Default Value:} " + sanitize(prop.getRawDefaultValue()), depr));
+ doc.println();
+ }
+ }
+
+ private String strike(String s, boolean isDeprecated) {
+ return (isDeprecated ? "\\sout{" : "") + s + (isDeprecated ? "}" : "");
+ }
+
+ @Override
+ void propertyTypeDescriptions() {
+ for (PropertyType type : PropertyType.values()) {
+ if (type == PropertyType.PREFIX)
+ continue;
+ doc.println("\\subsection{" + sanitize(type.toString()) + "}");
+ doc.println(sanitize(type.getFormatDescription()));
+ doc.println();
+ }
+ }
+
+ @Override
+ String sanitize(String str) {
+ str = str.replace("\\", "\\textbackslash{}");
+ str = str.replace("~", "\\textasciitilde{}");
+ str = str.replace("^", "\\textasciicircum");
+
+ str = str.replace("&", "\\&");
+ str = str.replace("%", "\\%");
+ str = str.replace("$", "\\$");
+ str = str.replace("#", "\\#");
+ str = str.replace("_", "\\_");
+ str = str.replace("{", "\\{");
+ str = str.replace("}", "\\}");
+
+ str = str.replaceAll("(?:\r\n|\r|\n)", "\\\\\\\\\n");
+ return str;
+ }
+ }
+
+ private static Logger log = Logger.getLogger(ConfigurationDocGen.class);
+
+ private PrintStream doc;
+
+ private final ArrayList<Property> prefixes;
+
+ private final TreeMap<String,Property> sortedProps;
+
+ ConfigurationDocGen(PrintStream doc) {
+ this.doc = doc;
+ this.prefixes = new ArrayList<Property>();
+ this.sortedProps = new TreeMap<String,Property>();
+
+ for (Property prop : Property.values()) {
+ if (prop.isExperimental())
+ continue;
+
+ if (prop.getType() == PropertyType.PREFIX)
+ this.prefixes.add(prop);
+ else
+ this.sortedProps.put(prop.getKey(), prop);
+ }
+ }
+
+ private void appendResource(String resourceName) {
+ InputStream data = ConfigurationDocGen.class.getResourceAsStream(resourceName);
+ if (data != null) {
+ byte[] buffer = new byte[1024];
+ int n;
+ try {
+ while ((n = data.read(buffer)) > 0)
- doc.print(new String(buffer, 0, n, Constants.UTF8));
++ doc.print(new String(buffer, 0, n, UTF_8));
+ } catch (IOException e) {
+ e.printStackTrace();
+ return;
+ } finally {
+ try {
+ data.close();
+ } catch (IOException ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }
+
+ private String isZooKeeperMutable(Property prop) {
+ if (!Property.isValidZooPropertyKey(prop.getKey()))
+ return "no";
+ if (Property.isFixedZooPropertyKey(prop))
+ return "yes but requires restart of the " + prop.getKey().split("[.]")[0];
+ return "yes";
+ }
+
+ void generateHtml() {
+ new HTML().generate();
+ }
+
+ void generateLaTeX() {
+ new LaTeX().generate();
+ }
+
+ /*
+ * Generate documentation for conf/accumulo-site.xml file usage
+ */
+ public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
+ if (args.length == 2 && args[0].equals("--generate-html")) {
- new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateHtml();
++ new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateHtml();
+ } else if (args.length == 2 && args[0].equals("--generate-latex")) {
- new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateLaTeX();
++ new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateLaTeX();
+ } else {
+ throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index ca769cb,7c4f66e..437590f
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@@ -16,11 -16,10 +16,11 @@@
*/
package org.apache.accumulo.core.data;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.io.Serializable;
+import java.nio.ByteBuffer;
- import org.apache.accumulo.core.Constants;
-
public class ArrayByteSequence extends ByteSequence implements Serializable {
private static final long serialVersionUID = 1L;
@@@ -48,22 -47,9 +48,22 @@@
}
public ArrayByteSequence(String s) {
- this(s.getBytes(Constants.UTF8));
+ this(s.getBytes(UTF_8));
}
+ public ArrayByteSequence(ByteBuffer buffer) {
+ this.length = buffer.remaining();
+
+ if (buffer.hasArray()) {
+ this.data = buffer.array();
+ this.offset = buffer.position();
+ } else {
+ this.data = new byte[length];
+ this.offset = 0;
+ buffer.get(data);
+ }
+ }
+
@Override
public byte byteAt(int i) {
[30/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
index 1226806,0000000..10cf224
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
@@@ -1,617 -1,0 +1,619 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.ServerClient;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
+import org.apache.accumulo.trace.instrument.TraceExecutorService;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+
+/*
+ * Bulk import makes requests of tablet servers, and those requests can take a
+ * long time. Our communications to the tablet server may fail, so we won't know
+ * the status of the request. The master will repeat failed requests so now
+ * there are multiple requests to the tablet server. The tablet server will not
+ * execute the request multiple times, so long as the marker it wrote in the
+ * metadata table stays there. The master needs to know when all requests have
+ * finished so it can remove the markers. Did it start? Did it finish? We can see
+ * that *a* request completed by seeing the flag written into the metadata
+ * table, but we won't know if some other rogue thread is still waiting to start
+ * a thread and repeat the operation.
+ *
+ * The master can ask the tablet server if it has any requests still running.
+ * Except the tablet server might have some thread about to start a request, but
+ * before it has made any bookkeeping about the request. To prevent problems
+ * like this, an Arbitrator is used. Before starting any new request, the tablet
+ * server checks the Arbitrator to see if the request is still valid.
+ *
+ */
+
+public class BulkImport extends MasterRepo {
+ public static final String FAILURES_TXT = "failures.txt";
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger log = Logger.getLogger(BulkImport.class);
+
+ private String tableId;
+ private String sourceDir;
+ private String errorDir;
+ private boolean setTime;
+
+ public BulkImport(String tableId, String sourceDir, String errorDir, boolean setTime) {
+ this.tableId = tableId;
+ this.sourceDir = sourceDir;
+ this.errorDir = errorDir;
+ this.setTime = setTime;
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+ if (!Utils.getReadLock(tableId, tid).tryLock())
+ return 100;
+
+ Instance instance = HdfsZooInstance.getInstance();
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) == TableState.ONLINE) {
+ long reserve1, reserve2;
+ reserve1 = reserve2 = Utils.reserveHdfsDirectory(sourceDir, tid);
+ if (reserve1 == 0)
+ reserve2 = Utils.reserveHdfsDirectory(errorDir, tid);
+ return reserve2;
+ } else {
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.OFFLINE, null);
+ }
+ }
+
+ @Override
+ //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ log.debug(" tid " + tid + " sourceDir " + sourceDir);
+
+ Utils.getReadLock(tableId, tid).lock();
+
+ // check that the error directory exists and is empty
+ VolumeManager fs = master.getFileSystem();
+
+ Path errorPath = new Path(errorDir);
+ FileStatus errorStatus = null;
+ try {
+ errorStatus = fs.getFileStatus(errorPath);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ if (errorStatus == null)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ + " does not exist");
+ if (!errorStatus.isDir())
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ + " is not a directory");
+ if (fs.listStatus(errorPath).length != 0)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ + " is not empty");
+
+ ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid);
+
+ // move the files into the directory
+ try {
+ String bulkDir = prepareBulkImport(fs, sourceDir, tableId);
+ log.debug(" tid " + tid + " bulkDir " + bulkDir);
+ return new LoadFiles(tableId, sourceDir, bulkDir, errorDir, setTime);
+ } catch (IOException ex) {
+ log.error("error preparing the bulk import directory", ex);
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_INPUT_DIRECTORY, sourceDir + ": "
+ + ex);
+ }
+ }
+
+ private Path createNewBulkDir(VolumeManager fs, String tableId) throws IOException {
+ Path tempPath = fs.matchingFileSystem(new Path(sourceDir), ServerConstants.getTablesDirs());
+ if (tempPath == null)
+ throw new IllegalStateException(sourceDir + " is not in a known namespace");
+
+ String tableDir = tempPath.toString();
+ if (tableDir == null)
+ throw new IllegalStateException(sourceDir + " is not in a known namespace");
+ Path directory = new Path(tableDir + "/" + tableId);
+ fs.mkdirs(directory);
+
+ // only one should be able to create the lock file
+ // the purpose of the lock file is to avoid a race
+ // condition between the call to fs.exists() and
+ // fs.mkdirs()... if only hadoop had a mkdir() function
+ // that failed when the dir existed
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
+ while (true) {
+ Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
+ if (fs.exists(newBulkDir)) // sanity check
+ throw new IllegalStateException("Dir exist when it should not " + newBulkDir);
+ if (fs.mkdirs(newBulkDir))
+ return newBulkDir;
+ log.warn("Failed to create " + newBulkDir + " for unknown reason");
+
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ //TODO Remove deprecation warning suppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ private String prepareBulkImport(VolumeManager fs, String dir, String tableId) throws IOException {
+ Path bulkDir = createNewBulkDir(fs, tableId);
+
+ MetadataTableUtil.addBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
+
+ Path dirPath = new Path(dir);
+ FileStatus[] mapFiles = fs.listStatus(dirPath);
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
+ for (FileStatus fileStatus : mapFiles) {
+ String sa[] = fileStatus.getPath().getName().split("\\.");
+ String extension = "";
+ if (sa.length > 1) {
+ extension = sa[sa.length - 1];
+
+ if (!FileOperations.getValidExtensions().contains(extension)) {
+ log.warn(fileStatus.getPath() + " does not have a valid extension, ignoring");
+ continue;
+ }
+ } else {
+ // assume it is a map file
+ extension = Constants.MAPFILE_EXTENSION;
+ }
+
+ if (extension.equals(Constants.MAPFILE_EXTENSION)) {
+ if (!fileStatus.isDir()) {
+ log.warn(fileStatus.getPath() + " is not a map file, ignoring");
+ continue;
+ }
+
+ if (fileStatus.getPath().getName().equals("_logs")) {
+ log.info(fileStatus.getPath() + " is probably a log directory from a map/reduce task, skipping");
+ continue;
+ }
+ try {
+ FileStatus dataStatus = fs.getFileStatus(new Path(fileStatus.getPath(), MapFile.DATA_FILE_NAME));
+ if (dataStatus.isDir()) {
+ log.warn(fileStatus.getPath() + " is not a map file, ignoring");
+ continue;
+ }
+ } catch (FileNotFoundException fnfe) {
+ log.warn(fileStatus.getPath() + " is not a map file, ignoring");
+ continue;
+ }
+ }
+
+ String newName = "I" + namer.getNextName() + "." + extension;
+ Path newPath = new Path(bulkDir, newName);
+ try {
+ fs.rename(fileStatus.getPath(), newPath);
+ log.debug("Moved " + fileStatus.getPath() + " to " + newPath);
+ } catch (IOException E1) {
+ log.error("Could not move: " + fileStatus.getPath().toString() + " " + E1.getMessage());
+ }
+ }
+ return bulkDir.toString();
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ // unreserve source/error directories
+ Utils.unreserveHdfsDirectory(sourceDir, tid);
+ Utils.unreserveHdfsDirectory(errorDir, tid);
+ Utils.getReadLock(tableId, tid).unlock();
+ }
+}
+
+class CleanUpBulkImport extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger log = Logger.getLogger(CleanUpBulkImport.class);
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String error;
+
+ public CleanUpBulkImport(String tableId, String source, String bulk, String error) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.error = error;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ log.debug("removing the bulk processing flag file in " + bulk);
+ Path bulkDir = new Path(bulk);
+ MetadataTableUtil.removeBulkLoadInProgressFlag("/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
+ MetadataTableUtil.addDeleteEntry(tableId, bulkDir.toString());
+ log.debug("removing the metadata table markers for loaded files");
+ Connector conn = master.getConnector();
+ MetadataTableUtil.removeBulkLoadEntries(conn, tableId, tid);
+ log.debug("releasing HDFS reservations for " + source + " and " + error);
+ Utils.unreserveHdfsDirectory(source, tid);
+ Utils.unreserveHdfsDirectory(error, tid);
+ Utils.getReadLock(tableId, tid).unlock();
+ log.debug("completing bulk import transaction " + tid);
+ ZooArbitrator.cleanup(Constants.BULK_ARBITRATOR_TYPE, tid);
+ return null;
+ }
+}
+
+class CompleteBulkImport extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String error;
+
+ public CompleteBulkImport(String tableId, String source, String bulk, String error) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.error = error;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ ZooArbitrator.stop(Constants.BULK_ARBITRATOR_TYPE, tid);
+ return new CopyFailed(tableId, source, bulk, error);
+ }
+}
+
+class CopyFailed extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String error;
+
+ public CopyFailed(String tableId, String source, String bulk, String error) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.error = error;
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+ Set<TServerInstance> finished = new HashSet<TServerInstance>();
+ Set<TServerInstance> running = master.onlineTabletServers();
+ for (TServerInstance server : running) {
+ try {
+ TServerConnection client = master.getConnection(server);
+ if (client != null && !client.isActive(tid))
+ finished.add(server);
+ } catch (TException ex) {
+ log.info("Ignoring error trying to check on tid " + tid + " from server " + server + ": " + ex);
+ }
+ }
+ if (finished.containsAll(running))
+ return 0;
+ return 500;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ // This needs to execute after the arbiter is stopped
+
+ VolumeManager fs = master.getFileSystem();
+
+ if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
+ return new CleanUpBulkImport(tableId, source, bulk, error);
+
+ HashMap<FileRef,String> failures = new HashMap<FileRef,String>();
+ HashMap<FileRef,String> loadedFailures = new HashMap<FileRef,String>();
+
+ FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
- BufferedReader in = new BufferedReader(new InputStreamReader(failFile, Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(failFile, UTF_8));
+ try {
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ Path path = new Path(line);
+ if (!fs.exists(new Path(error, path.getName())))
+ failures.put(new FileRef(line, path), line);
+ }
+ } finally {
+ failFile.close();
+ }
+
+ /*
+ * I thought I could move files that have no file references in the table. However its possible a clone references a file. Therefore only move files that
+ * have no loaded markers.
+ */
+
+ // determine which failed files were loaded
+ Connector conn = master.getConnector();
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ FileRef loadedFile = new FileRef(fs, entry.getKey());
+ String absPath = failures.remove(loadedFile);
+ if (absPath != null) {
+ loadedFailures.put(loadedFile, absPath);
+ }
+ }
+ }
+
+ // move failed files that were not loaded
+ for (String failure : failures.values()) {
+ Path orig = new Path(failure);
+ Path dest = new Path(error, orig.getName());
+ fs.rename(orig, dest);
+ log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
+ }
+
+ if (loadedFailures.size() > 0) {
+ DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
+ + Constants.ZBULK_FAILED_COPYQ);
+
+ HashSet<String> workIds = new HashSet<String>();
+
+ for (String failure : loadedFailures.values()) {
+ Path orig = new Path(failure);
+ Path dest = new Path(error, orig.getName());
+
+ if (fs.exists(dest))
+ continue;
+
- bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
++ bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(UTF_8));
+ workIds.add(orig.getName());
+ log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
+ }
+
+ bifCopyQueue.waitUntilDone(workIds);
+ }
+
+ fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
+ return new CleanUpBulkImport(tableId, source, bulk, error);
+ }
+
+}
+
+class LoadFiles extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private static ExecutorService threadPool = null;
+ private static final Logger log = Logger.getLogger(BulkImport.class);
+
+ private String tableId;
+ private String source;
+ private String bulk;
+ private String errorDir;
+ private boolean setTime;
+
+ public LoadFiles(String tableId, String source, String bulk, String errorDir, boolean setTime) {
+ this.tableId = tableId;
+ this.source = source;
+ this.bulk = bulk;
+ this.errorDir = errorDir;
+ this.setTime = setTime;
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+ if (master.onlineTabletServers().size() == 0)
+ return 500;
+ return 0;
+ }
+
+ private static synchronized ExecutorService getThreadPool(Master master) {
+ if (threadPool == null) {
+ int threadPoolSize = master.getSystemConfiguration().getCount(Property.MASTER_BULK_THREADPOOL_SIZE);
+ ThreadPoolExecutor pool = new SimpleThreadPool(threadPoolSize, "bulk import");
+ pool.allowCoreThreadTimeOut(true);
+ threadPool = new TraceExecutorService(pool);
+ }
+ return threadPool;
+ }
+
+ @Override
+ public Repo<Master> call(final long tid, final Master master) throws Exception {
+ ExecutorService executor = getThreadPool(master);
+ final SiteConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ VolumeManager fs = master.getFileSystem();
+ List<FileStatus> files = new ArrayList<FileStatus>();
+ for (FileStatus entry : fs.listStatus(new Path(bulk))) {
+ files.add(entry);
+ }
+ log.debug("tid " + tid + " importing " + files.size() + " files");
+
+ Path writable = new Path(this.errorDir, ".iswritable");
+ if (!fs.createNewFile(writable)) {
+ // Maybe this is a re-try... clear the flag and try again
+ fs.delete(writable);
+ if (!fs.createNewFile(writable))
+ throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY,
+ "Unable to write to " + this.errorDir);
+ }
+ fs.delete(writable);
+
+ final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<String>());
+ for (FileStatus f : files)
+ filesToLoad.add(f.getPath().toString());
+
+ final int RETRIES = Math.max(1, conf.getCount(Property.MASTER_BULK_RETRIES));
+ for (int attempt = 0; attempt < RETRIES && filesToLoad.size() > 0; attempt++) {
+ List<Future<List<String>>> results = new ArrayList<Future<List<String>>>();
+
+ if (master.onlineTabletServers().size() == 0)
+ log.warn("There are no tablet server to process bulk import, waiting (tid = " + tid + ")");
+
+ while (master.onlineTabletServers().size() == 0) {
+ UtilWaitThread.sleep(500);
+ }
+
+ // Use the threadpool to assign files one-at-a-time to the server
+ final List<String> loaded = Collections.synchronizedList(new ArrayList<String>());
+ for (final String file : filesToLoad) {
+ results.add(executor.submit(new Callable<List<String>>() {
+ @Override
+ public List<String> call() {
+ List<String> failures = new ArrayList<String>();
+ ClientService.Client client = null;
+ String server = null;
+ try {
+ // get a connection to a random tablet server, do not prefer cached connections because
+ // this is running on the master and there are lots of connections to tablet servers
+ // serving the metadata tablets
+ long timeInMillis = master.getConfiguration().getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
+ Pair<String,Client> pair = ServerClient.getConnection(master.getInstance(), false, timeInMillis);
+ client = pair.getSecond();
+ server = pair.getFirst();
+ List<String> attempt = Collections.singletonList(file);
+ log.debug("Asking " + pair.getFirst() + " to bulk import " + file);
+ List<String> fail = client.bulkImportFiles(Tracer.traceInfo(), SystemCredentials.get().toThrift(master.getInstance()), tid, tableId, attempt,
+ errorDir, setTime);
+ if (fail.isEmpty()) {
+ loaded.add(file);
+ } else {
+ failures.addAll(fail);
+ }
+ } catch (Exception ex) {
+ log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
+ } finally {
+ ServerClient.close(client);
+ }
+ return failures;
+ }
+ }));
+ }
+ Set<String> failures = new HashSet<String>();
+ for (Future<List<String>> f : results)
+ failures.addAll(f.get());
+ filesToLoad.removeAll(loaded);
+ if (filesToLoad.size() > 0) {
+ log.debug("tid " + tid + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
+ UtilWaitThread.sleep(100);
+ }
+ }
+
+ FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
- BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, Constants.UTF8));
++ BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8));
+ try {
+ for (String f : filesToLoad) {
+ out.write(f);
+ out.write("\n");
+ }
+ } finally {
+ out.close();
+ }
+
+ // return the next step, which will perform cleanup
+ return new CompleteBulkImport(tableId, source, bulk, errorDir);
+ }
+
+ static String sampleList(Collection<?> potentiallyLongList, int max) {
+ StringBuffer result = new StringBuffer();
+ result.append("[");
+ int i = 0;
+ for (Object obj : potentiallyLongList) {
+ result.append(obj);
+ if (i >= max) {
+ result.append("...");
+ break;
+ } else {
+ result.append(", ");
+ }
+ i++;
+ }
+ if (i < max)
+ result.delete(result.length() - 2, result.length());
+ result.append("]");
+ return result.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
index ebceaa0,0000000..fd2441c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
@@@ -1,105 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+class FinishCancelCompaction extends MasterRepo {
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+
+ public FinishCancelCompaction(String tableId) {
+ this.tableId = tableId;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ Utils.getReadLock(tableId, tid).unlock();
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+
+ }
+}
+
+/**
+ *
+ */
+public class CancelCompactions extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+ private String namespaceId;
+
+ public CancelCompactions(String tableId) {
+ this.tableId = tableId;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT_CANCEL)
+ + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT_CANCEL);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ String zCompactID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+ String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ byte[] currentValue = zoo.getData(zCompactID, null);
+
- String cvs = new String(currentValue, Constants.UTF8);
++ String cvs = new String(currentValue, UTF_8);
+ String[] tokens = cvs.split(",");
+ final long flushID = Long.parseLong(tokens[0]);
+
+ zoo.mutate(zCancelID, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- long cid = Long.parseLong(new String(currentValue, Constants.UTF8));
++ long cid = Long.parseLong(new String(currentValue, UTF_8));
+
+ if (cid < flushID)
- return Long.toString(flushID).getBytes(Constants.UTF8);
++ return Long.toString(flushID).getBytes(UTF_8);
+ else
- return Long.toString(cid).getBytes(Constants.UTF8);
++ return Long.toString(cid).getBytes(UTF_8);
+ }
+ });
+
+ return new FinishCancelCompaction(tableId);
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ Utils.unreserveNamespace(namespaceId, tid, false);
+ Utils.unreserveTable(tableId, tid, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index a643ccb,0000000..fbaa70d
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@@ -1,400 -1,0 +1,402 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+class CompactionDriver extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private long compactId;
+ private String tableId;
+ private byte[] startRow;
+ private byte[] endRow;
+ private String namespaceId;
+
+ public CompactionDriver(long compactId, String tableId, byte[] startRow, byte[] endRow) {
+
+ this.compactId = compactId;
+ this.tableId = tableId;
+ this.startRow = startRow;
+ this.endRow = endRow;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+
+ String zCancelID = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ if (Long.parseLong(new String(zoo.getData(zCancelID, null))) >= compactId) {
+ // compaction was canceled
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, "Compaction canceled");
+ }
+
+ MapCounter<TServerInstance> serversToFlush = new MapCounter<TServerInstance>();
+ Connector conn = master.getConnector();
+
+ Scanner scanner;
+
+ if (tableId.equals(MetadataTable.ID)) {
+ scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ } else {
+ scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+ Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange();
+ scanner.setRange(range);
+ }
+
+ TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner);
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+
+ long t1 = System.currentTimeMillis();
+ RowIterator ri = new RowIterator(scanner);
+
+ int tabletsToWaitFor = 0;
+ int tabletCount = 0;
+
+ while (ri.hasNext()) {
+ Iterator<Entry<Key,Value>> row = ri.next();
+ long tabletCompactID = -1;
+
+ TServerInstance server = null;
+
+ Entry<Key,Value> entry = null;
+ while (row.hasNext()) {
+ entry = row.next();
+ Key key = entry.getKey();
+
+ if (TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ tabletCompactID = Long.parseLong(entry.getValue().toString());
+
+ if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily()))
+ server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
+ }
+
+ if (tabletCompactID < compactId) {
+ tabletsToWaitFor++;
+ if (server != null)
+ serversToFlush.increment(server, 1);
+ }
+
+ tabletCount++;
+
+ Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
+ if (tabletEndRow == null || (endRow != null && tabletEndRow.compareTo(new Text(endRow)) >= 0))
+ break;
+ }
+
+ long scanTime = System.currentTimeMillis() - t1;
+
+ Instance instance = master.getInstance();
+ Tables.clearCache(instance);
+ if (tabletCount == 0 && !Tables.exists(instance, tableId))
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
+
+ if (serversToFlush.size() == 0 && Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
+
+ if (tabletsToWaitFor == 0)
+ return 0;
+
+ for (TServerInstance tsi : serversToFlush.keySet()) {
+ try {
+ final TServerConnection server = master.getConnection(tsi);
+ if (server != null)
+ server.compact(master.getMasterLock(), tableId, startRow, endRow);
+ } catch (TException ex) {
+ Logger.getLogger(CompactionDriver.class).error(ex.toString());
+ }
+ }
+
+ long sleepTime = 500;
+
+ if (serversToFlush.size() > 0)
+ sleepTime = Collections.max(serversToFlush.values()) * sleepTime; // make wait time depend on the server with the most to
+ // compact
+
+ sleepTime = Math.max(2 * scanTime, sleepTime);
+
+ sleepTime = Math.min(sleepTime, 30000);
+
+ return sleepTime;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+ CompactRange.removeIterators(tid, tableId);
+ Utils.getReadLock(tableId, tid).unlock();
+ Utils.getReadLock(namespaceId, tid).unlock();
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+
+ }
+
+}
+
+public class CompactRange extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+ private byte[] startRow;
+ private byte[] endRow;
+ private byte[] iterators;
+ private String namespaceId;
+
+ public static class CompactionIterators implements Writable {
+ byte[] startRow;
+ byte[] endRow;
+ List<IteratorSetting> iterators;
+
+ public CompactionIterators(byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) {
+ this.startRow = startRow;
+ this.endRow = endRow;
+ this.iterators = iterators;
+ }
+
+ public CompactionIterators() {
+ startRow = null;
+ endRow = null;
+ iterators = Collections.emptyList();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(startRow != null);
+ if (startRow != null) {
+ out.writeInt(startRow.length);
+ out.write(startRow);
+ }
+
+ out.writeBoolean(endRow != null);
+ if (endRow != null) {
+ out.writeInt(endRow.length);
+ out.write(endRow);
+ }
+
+ out.writeInt(iterators.size());
+ for (IteratorSetting is : iterators) {
+ is.write(out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ if (in.readBoolean()) {
+ startRow = new byte[in.readInt()];
+ in.readFully(startRow);
+ } else {
+ startRow = null;
+ }
+
+ if (in.readBoolean()) {
+ endRow = new byte[in.readInt()];
+ in.readFully(endRow);
+ } else {
+ endRow = null;
+ }
+
+ int num = in.readInt();
+ iterators = new ArrayList<IteratorSetting>(num);
+
+ for (int i = 0; i < num; i++) {
+ iterators.add(new IteratorSetting(in));
+ }
+ }
+
+ public Text getEndRow() {
+ if (endRow == null)
+ return null;
+ return new Text(endRow);
+ }
+
+ public Text getStartRow() {
+ if (startRow == null)
+ return null;
+ return new Text(startRow);
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+ }
+
+ public CompactRange(String tableId, byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) throws ThriftTableOperationException {
+ this.tableId = tableId;
+ this.startRow = startRow.length == 0 ? null : startRow;
+ this.endRow = endRow.length == 0 ? null : endRow;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+
+ if (iterators.size() > 0) {
+ this.iterators = WritableUtils.toByteArray(new CompactionIterators(this.startRow, this.endRow, iterators));
+ } else {
+ iterators = null;
+ }
+
+ if (this.startRow != null && this.endRow != null && new Text(startRow).compareTo(new Text(endRow)) >= 0)
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.BAD_RANGE,
+ "start row must be less than end row");
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.COMPACT)
+ + Utils.reserveTable(tableId, tid, false, true, TableOperation.COMPACT);
+ }
+
+ @Override
+ public Repo<Master> call(final long tid, Master environment) throws Exception {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ byte[] cid;
+ try {
+ cid = zoo.mutate(zTablePath, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- String cvs = new String(currentValue, Constants.UTF8);
++ String cvs = new String(currentValue, UTF_8);
+ String[] tokens = cvs.split(",");
+ long flushID = Long.parseLong(tokens[0]);
+ flushID++;
+
+ String txidString = String.format("%016x", tid);
+
+ for (int i = 1; i < tokens.length; i++) {
+ if (tokens[i].startsWith(txidString))
+ continue; // skip self
+
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.OTHER,
+ "Another compaction with iterators is running");
+ }
+
+ StringBuilder encodedIterators = new StringBuilder();
+
+ if (iterators != null) {
+ Hex hex = new Hex();
+ encodedIterators.append(",");
+ encodedIterators.append(txidString);
+ encodedIterators.append("=");
- encodedIterators.append(new String(hex.encode(iterators), Constants.UTF8));
++ encodedIterators.append(new String(hex.encode(iterators), UTF_8));
+ }
+
- return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++ return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
+ }
+ });
+
- return new CompactionDriver(Long.parseLong(new String(cid, Constants.UTF8).split(",")[0]), tableId, startRow, endRow);
++ return new CompactionDriver(Long.parseLong(new String(cid, UTF_8).split(",")[0]), tableId, startRow, endRow);
+ } catch (NoNodeException nne) {
+ throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
+ }
+
+ }
+
+ static void removeIterators(final long txid, String tableId) throws Exception {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ zoo.mutate(zTablePath, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- String cvs = new String(currentValue, Constants.UTF8);
++ String cvs = new String(currentValue, UTF_8);
+ String[] tokens = cvs.split(",");
+ long flushID = Long.parseLong(tokens[0]);
+
+ String txidString = String.format("%016x", txid);
+
+ StringBuilder encodedIterators = new StringBuilder();
+ for (int i = 1; i < tokens.length; i++) {
+ if (tokens[i].startsWith(txidString))
+ continue;
+ encodedIterators.append(",");
+ encodedIterators.append(tokens[i]);
+ }
+
- return (Long.toString(flushID) + encodedIterators).getBytes(Constants.UTF8);
++ return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
+ }
+ });
+
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ try {
+ removeIterators(tid, tableId);
+ } finally {
+ Utils.unreserveNamespace(namespaceId, tid, false);
+ Utils.unreserveTable(tableId, tid, false);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index eeb9b16,0000000..037455c
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@@ -1,316 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+class ExportInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public String tableName;
+ public String tableID;
+ public String exportDir;
+ public String namespaceID;
+}
+
+class WriteExportFiles extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private final ExportInfo tableInfo;
+
+ WriteExportFiles(ExportInfo tableInfo) {
+ this.tableInfo = tableInfo;
+ }
+
+ private void checkOffline(Connector conn) throws Exception {
+ if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
+ Tables.clearCache(conn.getInstance());
+ if (Tables.getTableState(conn.getInstance(), tableInfo.tableID) != TableState.OFFLINE) {
+ throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
+ "Table is not offline");
+ }
+ }
+ }
+
+ @Override
+ public long isReady(long tid, Master master) throws Exception {
+
+ long reserved = Utils.reserveNamespace(tableInfo.namespaceID, tid, false, true, TableOperation.EXPORT)
+ + Utils.reserveTable(tableInfo.tableID, tid, false, true, TableOperation.EXPORT);
+ if (reserved > 0)
+ return reserved;
+
+ Connector conn = master.getConnector();
+
+ checkOffline(conn);
+
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.setRange(new KeyExtent(new Text(tableInfo.tableID), null, null).toMetadataRange());
+
+ // scan for locations
+ metaScanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ metaScanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
+
+ if (metaScanner.iterator().hasNext()) {
+ return 500;
+ }
+
+ // use the same range to check for walogs that we used to check for hosted (or future hosted) tablets
+ // this is done as a separate scan after we check for locations, because walogs are okay only if there is no location
+ metaScanner.clearColumns();
+ metaScanner.fetchColumnFamily(LogColumnFamily.NAME);
+
+ if (metaScanner.iterator().hasNext()) {
+ throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
+ "Write ahead logs found for table");
+ }
+
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ Connector conn = master.getConnector();
+
+ try {
+ exportTable(master.getFileSystem(), conn, tableInfo.tableName, tableInfo.tableID, tableInfo.exportDir);
+ } catch (IOException ioe) {
+ throw new ThriftTableOperationException(tableInfo.tableID, tableInfo.tableName, TableOperation.EXPORT, TableOperationExceptionType.OTHER,
+ "Failed to create export files " + ioe.getMessage());
+ }
+ Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
+ Utils.unreserveTable(tableInfo.tableID, tid, false);
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveNamespace(tableInfo.namespaceID, tid, false);
+ Utils.unreserveTable(tableInfo.tableID, tid, false);
+ }
+
+ public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
+
+ fs.mkdirs(new Path(exportDir));
+ Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
+
+ FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
+ ZipOutputStream zipOut = new ZipOutputStream(fileOut);
+ BufferedOutputStream bufOut = new BufferedOutputStream(zipOut);
+ DataOutputStream dataOut = new DataOutputStream(bufOut);
+
+ try {
+
+ zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_INFO_FILE));
- OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++ OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
+ osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.VERSION + "\n");
+ osw.append("srcInstanceName:" + conn.getInstance().getInstanceName() + "\n");
+ osw.append("srcInstanceID:" + conn.getInstance().getInstanceID() + "\n");
+ osw.append("srcZookeepers:" + conn.getInstance().getZooKeepers() + "\n");
+ osw.append("srcTableName:" + tableName + "\n");
+ osw.append("srcTableID:" + tableID + "\n");
+ osw.append(ExportTable.DATA_VERSION_PROP + ":" + ServerConstants.DATA_VERSION + "\n");
+ osw.append("srcCodeVersion:" + Constants.VERSION + "\n");
+
+ osw.flush();
+ dataOut.flush();
+
+ exportConfig(conn, tableID, zipOut, dataOut);
+ dataOut.flush();
+
+ Map<String,String> uniqueFiles = exportMetadata(fs, conn, tableID, zipOut, dataOut);
+
+ dataOut.close();
+ dataOut = null;
+
+ createDistcpFile(fs, exportDir, exportMetaFilePath, uniqueFiles);
+
+ } finally {
+ if (dataOut != null)
+ dataOut.close();
+ }
+ }
+
+ private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException {
- BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), Constants.UTF8));
++ BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), UTF_8));
+
+ try {
+ for (String file : uniqueFiles.values()) {
+ distcpOut.append(file);
+ distcpOut.newLine();
+ }
+
+ distcpOut.append(exportMetaFilePath.toString());
+ distcpOut.newLine();
+
+ distcpOut.close();
+ distcpOut = null;
+
+ } finally {
+ if (distcpOut != null)
+ distcpOut.close();
+ }
+ }
+
+ private static Map<String,String> exportMetadata(VolumeManager fs, Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut)
+ throws IOException, TableNotFoundException {
+ zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_METADATA_FILE));
+
+ Map<String,String> uniqueFiles = new HashMap<String,String>();
+
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(metaScanner);
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(metaScanner);
+ metaScanner.setRange(new KeyExtent(new Text(tableID), null, null).toMetadataRange());
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ entry.getKey().write(dataOut);
+ entry.getValue().write(dataOut);
+
+ if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String path = fs.getFullPath(entry.getKey()).toString();
+ String tokens[] = path.split("/");
+ if (tokens.length < 1) {
+ throw new RuntimeException("Illegal path " + path);
+ }
+
+ String filename = tokens[tokens.length - 1];
+
+ String existingPath = uniqueFiles.get(filename);
+ if (existingPath == null) {
+ uniqueFiles.put(filename, path);
+ } else if (!existingPath.equals(path)) {
+ // make sure file names are unique, should only apply for tables with file names generated by Accumulo 1.3 and earlier
+ throw new IOException("Cannot export table with nonunique file names " + filename + ". Major compact table.");
+ }
+
+ }
+ }
+ return uniqueFiles;
+ }
+
+ private static void exportConfig(Connector conn, String tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws AccumuloException,
+ AccumuloSecurityException, TableNotFoundException, IOException {
+
+ DefaultConfiguration defaultConfig = AccumuloConfiguration.getDefaultConfiguration();
+ Map<String,String> siteConfig = conn.instanceOperations().getSiteConfiguration();
+ Map<String,String> systemConfig = conn.instanceOperations().getSystemConfiguration();
+
+ TableConfiguration tableConfig = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableID);
+
- OutputStreamWriter osw = new OutputStreamWriter(dataOut, Constants.UTF8);
++ OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
+
+ // only put props that are different than defaults and higher level configurations
+ zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));
+ for (Entry<String,String> prop : tableConfig) {
+ if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) {
+ Property key = Property.getPropertyByKey(prop.getKey());
+
+ if (key == null || !defaultConfig.get(key).equals(prop.getValue())) {
+ if (!prop.getValue().equals(siteConfig.get(prop.getKey())) && !prop.getValue().equals(systemConfig.get(prop.getKey()))) {
+ osw.append(prop.getKey() + "=" + prop.getValue() + "\n");
+ }
+ }
+ }
+ }
+
+ osw.flush();
+ }
+}
+
+public class ExportTable extends MasterRepo {
+ private static final long serialVersionUID = 1L;
+
+ private final ExportInfo tableInfo;
+
+ public ExportTable(String tableName, String tableId, String exportDir) {
+ tableInfo = new ExportInfo();
+ tableInfo.tableName = tableName;
+ tableInfo.exportDir = exportDir;
+ tableInfo.tableID = tableId;
+ Instance inst = HdfsZooInstance.getInstance();
+ tableInfo.namespaceID = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ return new WriteExportFiles(tableInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ }
+
+ public static final int VERSION = 1;
+
+ public static final String DATA_VERSION_PROP = "srcDataVersion";
+ public static final String EXPORT_VERSION_PROP = "exportVersion";
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 8de6bce,0000000..05ec263
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@@ -1,653 -1,0 +1,655 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.TableOperationsImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+class ImportedTableInfo implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public String exportDir;
+ public String user;
+ public String tableName;
+ public String tableId;
+ public String importDir;
+ public String namespaceId;
+}
+
+class FinishImportTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public FinishImportTable(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+
+ env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir, "mappings.txt"));
+
+ TableManager.getInstance().transitionTableState(tableInfo.tableId, TableState.ONLINE);
+
+ Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
+ Utils.unreserveTable(tableInfo.tableId, tid, true);
+
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+
+ env.getEventCoordinator().event("Imported table %s ", tableInfo.tableName);
+
+ Logger.getLogger(FinishImportTable.class).debug("Imported table " + tableInfo.tableId + " " + tableInfo.tableName);
+
+ return null;
+ }
+
+ @Override
+ public String getReturn() {
+ return tableInfo.tableId;
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {}
+
+}
+
+class MoveExportedFiles extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ MoveExportedFiles(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ try {
+ VolumeManager fs = master.getFileSystem();
+
+ Map<String,String> fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo);
+
+ for (String oldFileName : fileNameMappings.keySet()) {
+ if (!fs.exists(new Path(tableInfo.exportDir, oldFileName))) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "File referenced by exported table does not exists " + oldFileName);
+ }
+ }
+
+ FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
+
+ for (FileStatus fileStatus : files) {
+ String newName = fileNameMappings.get(fileStatus.getPath().getName());
+
+ if (newName != null)
+ fs.rename(fileStatus.getPath(), new Path(tableInfo.importDir, newName));
+ }
+
+ return new FinishImportTable(tableInfo);
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error renaming files " + ioe.getMessage());
+ }
+ }
+}
+
+class PopulateMetadataTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ PopulateMetadataTable(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ static Map<String,String> readMappingFile(VolumeManager fs, ImportedTableInfo tableInfo) throws Exception {
- BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(tableInfo.importDir, "mappings.txt")), UTF_8));
+
+ try {
+ Map<String,String> map = new HashMap<String,String>();
+
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split(":", 2);
+ map.put(sa[0], sa[1]);
+ }
+
+ return map;
+ } finally {
+ in.close();
+ }
+
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+
+ Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
+
+ BatchWriter mbw = null;
+ ZipInputStream zis = null;
+
+ try {
+ VolumeManager fs = master.getFileSystem();
+
+ mbw = master.getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ zis = new ZipInputStream(fs.open(path));
+
+ Map<String,String> fileNameMappings = readMappingFile(fs, tableInfo);
+
+ log.info("importDir is " + tableInfo.importDir);
+
+ // This is a directory already prefixed with proper volume information e.g. hdfs://localhost:8020/path/to/accumulo/tables/...
+ final String bulkDir = tableInfo.importDir;
+
+ final String[] tableDirs = ServerConstants.getTablesDirs();
+
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) {
+ DataInputStream in = new DataInputStream(new BufferedInputStream(zis));
+
+ Key key = new Key();
+ Value val = new Value();
+
+ Mutation m = null;
+ Text currentRow = null;
+ int dirCount = 0;
+
+ while (true) {
+ key.readFields(in);
+ val.readFields(in);
+
+ Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow();
+ Text metadataRow = new KeyExtent(new Text(tableInfo.tableId), endRow, null).getMetadataEntry();
+
+ Text cq;
+
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String oldName = new Path(key.getColumnQualifier().toString()).getName();
+ String newName = fileNameMappings.get(oldName);
+
+ if (newName == null) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "File " + oldName + " does not exist in import dir");
+ }
+
+ cq = new Text(bulkDir + "/" + newName);
+ } else {
+ cq = key.getColumnQualifier();
+ }
+
+ if (m == null) {
+ // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
- String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++ String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
+
+ // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
+ String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+
+ m = new Mutation(metadataRow);
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
+ currentRow = metadataRow;
+ }
+
+ if (!currentRow.equals(metadataRow)) {
+ mbw.addMutation(m);
+
+ // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator
- String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), Constants.UTF8);
++ String tabletDir = new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8);
+
+ // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
+ String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir);
+
+ m = new Mutation(metadataRow);
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8)));
+ }
+
+ m.put(key.getColumnFamily(), cq, val);
+
+ if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
+ mbw.addMutation(m);
+ break; // its the last column in the last row
+ }
+ }
+
+ break;
+ }
+ }
+
+ return new MoveExportedFiles(tableInfo);
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error reading " + path + " " + ioe.getMessage());
+ } finally {
+ if (zis != null) {
+ try {
+ zis.close();
+ } catch (IOException ioe) {
+ log.warn("Failed to close zip file ", ioe);
+ }
+ }
+
+ if (mbw != null) {
+ mbw.close();
+ }
+ }
+ }
+
+ /**
+ * Given options for tables (across multiple volumes), construct an absolute path using the unique name within the chosen volume
+ *
+ * @return An absolute, unique path for the imported table
+ */
+ protected String getClonedTabletDir(Master master, String[] tableDirs, String tabletDir) {
+ // We can try to spread out the tablet dirs across all volumes
+ String tableDir = master.getFileSystem().choose(tableDirs);
+
+ // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX
+ return tableDir + "/" + tableInfo.tableId + "/" + tabletDir;
+ }
+
+ @Override
+ public void undo(long tid, Master environment) throws Exception {
+ MetadataTableUtil.deleteTable(tableInfo.tableId, false, SystemCredentials.get(), environment.getMasterLock());
+ }
+}
+
+class MapImportFileNames extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ MapImportFileNames(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master environment) throws Exception {
+
+ Path path = new Path(tableInfo.importDir, "mappings.txt");
+
+ BufferedWriter mappingsWriter = null;
+
+ try {
+ VolumeManager fs = environment.getFileSystem();
+
+ fs.mkdirs(new Path(tableInfo.importDir));
+
+ FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir));
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
- mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), Constants.UTF8));
++ mappingsWriter = new BufferedWriter(new OutputStreamWriter(fs.create(path), UTF_8));
+
+ for (FileStatus fileStatus : files) {
+ String fileName = fileStatus.getPath().getName();
+ log.info("filename " + fileStatus.getPath().toString());
+ String sa[] = fileName.split("\\.");
+ String extension = "";
+ if (sa.length > 1) {
+ extension = sa[sa.length - 1];
+
+ if (!FileOperations.getValidExtensions().contains(extension)) {
+ continue;
+ }
+ } else {
+ // assume it is a map file
+ extension = Constants.MAPFILE_EXTENSION;
+ }
+
+ String newName = "I" + namer.getNextName() + "." + extension;
+
+ mappingsWriter.append(fileName);
+ mappingsWriter.append(':');
+ mappingsWriter.append(newName);
+ mappingsWriter.newLine();
+ }
+
+ mappingsWriter.close();
+ mappingsWriter = null;
+
+ return new PopulateMetadataTable(tableInfo);
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error writing mapping file " + path + " " + ioe.getMessage());
+ } finally {
+ if (mappingsWriter != null)
+ try {
+ mappingsWriter.close();
+ } catch (IOException ioe) {
+ log.warn("Failed to close " + path, ioe);
+ }
+ }
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ env.getFileSystem().deleteRecursively(new Path(tableInfo.importDir));
+ }
+}
+
+class CreateImportDir extends MasterRepo {
+ private static final Logger log = Logger.getLogger(CreateImportDir.class);
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ CreateImportDir(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+
+ UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
+
+ Path exportDir = new Path(tableInfo.exportDir);
+ String[] tableDirs = ServerConstants.getTablesDirs();
+
+ log.info("Looking for matching filesystem for " + exportDir + " from options " + Arrays.toString(tableDirs));
+ Path base = master.getFileSystem().matchingFileSystem(exportDir, tableDirs);
+ log.info("Chose base table directory of " + base);
+ Path directory = new Path(base, tableInfo.tableId);
+
+ Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName());
+
+ tableInfo.importDir = newBulkDir.toString();
+
+ log.info("Using import dir: " + tableInfo.importDir);
+
+ return new MapImportFileNames(tableInfo);
+ }
+}
+
+class ImportPopulateZookeeper extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ ImportPopulateZookeeper(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT);
+ }
+
+ private Map<String,String> getExportedProps(VolumeManager fs) throws Exception {
+
+ Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
+
+ try {
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ return TableOperationsImpl.getExportedProps(ns, path);
+ } catch (IOException ioe) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Error reading table props from " + path + " " + ioe.getMessage());
+ }
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ // reserve the table name in zookeeper or fail
+
+ Utils.tableNameLock.lock();
+ try {
+ // write tableName & tableId to zookeeper
+ Instance instance = HdfsZooInstance.getInstance();
+
+ Utils.checkTableDoesNotExist(instance, tableInfo.tableName, tableInfo.tableId, TableOperation.CREATE);
+
+ String namespace = Tables.qualify(tableInfo.tableName).getFirst();
+ String namespaceId = Namespaces.getNamespaceId(instance, namespace);
+ TableManager.getInstance().addTable(tableInfo.tableId, namespaceId, tableInfo.tableName, NodeExistsPolicy.OVERWRITE);
+
+ Tables.clearCache(instance);
+ } finally {
+ Utils.tableNameLock.unlock();
+ }
+
+ for (Entry<String,String> entry : getExportedProps(env.getFileSystem()).entrySet())
+ if (!TablePropUtil.setTableProperty(tableInfo.tableId, entry.getKey(), entry.getValue())) {
+ throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Invalid table property " + entry.getKey());
+ }
+
+ return new CreateImportDir(tableInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Instance instance = HdfsZooInstance.getInstance();
+ TableManager.getInstance().removeTable(tableInfo.tableId);
+ Utils.unreserveTable(tableInfo.tableId, tid, true);
+ Tables.clearCache(instance);
+ }
+}
+
+class ImportSetupPermissions extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public ImportSetupPermissions(ImportedTableInfo ti) {
+ this.tableInfo = ti;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ // give all table permissions to the creator
+ SecurityOperation security = AuditedSecurityOperation.getInstance();
+ for (TablePermission permission : TablePermission.values()) {
+ try {
+ security.grantTablePermission(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.user, tableInfo.tableId, permission, tableInfo.namespaceId);
+ } catch (ThriftSecurityException e) {
+ Logger.getLogger(ImportSetupPermissions.class).error(e.getMessage(), e);
+ throw e;
+ }
+ }
+
+ // setup permissions in zookeeper before table info in zookeeper
+ // this way concurrent users will not get a spurious permission denied
+ // error
+ return new ImportPopulateZookeeper(tableInfo);
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ AuditedSecurityOperation.getInstance().deleteTable(SystemCredentials.get().toThrift(env.getInstance()), tableInfo.tableId, tableInfo.namespaceId);
+ }
+}
+
+public class ImportTable extends MasterRepo {
+ private static final long serialVersionUID = 1L;
+
+ private ImportedTableInfo tableInfo;
+
+ public ImportTable(String user, String tableName, String exportDir, String namespaceId) {
+ tableInfo = new ImportedTableInfo();
+ tableInfo.tableName = tableName;
+ tableInfo.user = user;
+ tableInfo.exportDir = exportDir;
+ tableInfo.namespaceId = namespaceId;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid)
+ + Utils.reserveNamespace(tableInfo.namespaceId, tid, false, true, TableOperation.IMPORT);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master env) throws Exception {
+ checkVersions(env);
+
+ // first step is to reserve a table id.. if the machine fails during this step
+ // it is ok to retry... the only side effect is that a table id may not be used
+ // or skipped
+
+ // assuming only the master process is creating tables
+
+ Utils.idLock.lock();
+ try {
+ Instance instance = HdfsZooInstance.getInstance();
+ tableInfo.tableId = Utils.getNextTableId(tableInfo.tableName, instance);
+ return new ImportSetupPermissions(tableInfo);
+ } finally {
+ Utils.idLock.unlock();
+ }
+ }
+
+ public void checkVersions(Master env) throws ThriftTableOperationException {
+ Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
+
+ ZipInputStream zis = null;
+
+ try {
+ zis = new ZipInputStream(env.getFileSystem().open(path));
+
+ Integer exportVersion = null;
+ Integer dataVersion = null;
+
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_INFO_FILE)) {
- BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split(":", 2);
+ if (sa[0].equals(ExportTable.EXPORT_VERSION_PROP)) {
+ exportVersion = Integer.parseInt(sa[1]);
+ } else if (sa[0].equals(ExportTable.DATA_VERSION_PROP)) {
+ dataVersion = Integer.parseInt(sa[1]);
+ }
+ }
+
+ break;
+ }
+ }
+
+ zis.close();
+ zis = null;
+
+ if (exportVersion == null || exportVersion > ExportTable.VERSION)
+ throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Incompatible export version " + exportVersion);
+
+ if (dataVersion == null || dataVersion > ServerConstants.DATA_VERSION)
+ throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Incompatible data version " + exportVersion);
+
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ throw new ThriftTableOperationException(null, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
+ "Failed to read export metadata " + ioe.getMessage());
+ } finally {
+ if (zis != null)
+ try {
+ zis.close();
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage(), ioe);
+ }
+ }
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid);
+ Utils.unreserveNamespace(tableInfo.namespaceId, tid, false);
+ }
+}
[03/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
index e44d8df,0000000..49b5d70
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
@@@ -1,496 -1,0 +1,496 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests that Accumulo is outputting audit messages as expected. Since this is using MiniAccumuloCluster, it could take a while if we test everything in
+ * isolation. We test blocks of related operations, run the whole test in one MiniAccumulo instance, trying to clean up objects between each test. The
+ * MiniAccumuloClusterTest sets up the log4j stuff differently to an installed instance, instead piping everything through stdout and writing to a set location
+ * so we have to find the logs and grep the bits we need out.
+ */
+public class AuditMessageIT extends ConfigurableMacIT {
+
+ private static final String AUDIT_USER_1 = "AuditUser1";
+ private static final String AUDIT_USER_2 = "AuditUser2";
+ private static final String PASSWORD = "password";
+ private static final String OLD_TEST_TABLE_NAME = "apples";
+ private static final String NEW_TEST_TABLE_NAME = "oranges";
+ private static final String THIRD_TEST_TABLE_NAME = "pears";
+ private static final Authorizations auths = new Authorizations("private", "public");
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {
+ new File(cfg.getConfDir(), "auditLog.xml").delete();
+ }
+
+ // Must be static to survive Junit re-initialising the class every time.
+ private static String lastAuditTimestamp;
+ private Connector auditConnector;
+ private Connector conn;
+
+ private static ArrayList<String> findAuditMessage(ArrayList<String> input, String pattern) {
+ ArrayList<String> result = new ArrayList<String>();
+ for (String s : input) {
+ if (s.matches(".*" + pattern + ".*"))
+ result.add(s);
+ }
+ return result;
+ }
+
+ /**
+ * Returns a List of Audit messages that have been grep'd out of the MiniAccumuloCluster output.
+ *
+ * @param stepName
+ * A unique name for the test being executed, to identify the System.out messages.
+ * @return A List of the Audit messages, sorted (so in chronological order).
+ */
+ private ArrayList<String> getAuditMessages(String stepName) throws IOException {
+ // ACCUMULO-3144 Make sure we give the processes enough time to flush the write buffer
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted waiting for data to be flushed to output streams");
+ }
+
+ for (MiniAccumuloClusterImpl.LogWriter lw : getCluster().getLogWriters()) {
+ lw.flush();
+ }
+
+ // Grab the audit messages
+ System.out.println("Start of captured audit messages for step " + stepName);
+
+ ArrayList<String> result = new ArrayList<String>();
+ for (File file : getCluster().getConfig().getLogDir().listFiles()) {
+ // We want to grab the files called .out
+ if (file.getName().contains(".out") && file.isFile() && file.canRead()) {
- LineIterator it = FileUtils.lineIterator(file, Constants.UTF8.name());
++ LineIterator it = FileUtils.lineIterator(file, UTF_8.name());
+ try {
+ while (it.hasNext()) {
+ String line = it.nextLine();
+ if (line.matches(".* \\[" + AuditedSecurityOperation.AUDITLOG + "\\s*\\].*")) {
+ // Only include the message if startTimestamp is null. or the message occurred after the startTimestamp value
+ if ((lastAuditTimestamp == null) || (line.substring(0, 23).compareTo(lastAuditTimestamp) > 0))
+ result.add(line);
+ }
+ }
+ } finally {
+ LineIterator.closeQuietly(it);
+ }
+ }
+ }
+ Collections.sort(result);
+
+ for (String s : result) {
+ System.out.println(s);
+ }
+ System.out.println("End of captured audit messages for step " + stepName);
+ if (result.size() > 0)
+ lastAuditTimestamp = (result.get(result.size() - 1)).substring(0, 23);
+
+ return result;
+ }
+
+ private void grantEverySystemPriv(Connector conn, String user) throws AccumuloSecurityException, AccumuloException {
+ SystemPermission[] arrayOfP = new SystemPermission[] {SystemPermission.SYSTEM, SystemPermission.ALTER_TABLE, SystemPermission.ALTER_USER,
+ SystemPermission.CREATE_TABLE, SystemPermission.CREATE_USER, SystemPermission.DROP_TABLE, SystemPermission.DROP_USER};
+ for (SystemPermission p : arrayOfP) {
+ conn.securityOperations().grantSystemPermission(user, p);
+ }
+ }
+
+ @Before
+ public void resetInstance() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException {
+ conn = getConnector();
+
+ // I don't want to recreate the instance for every test since it will take ages.
+ // If we run every test as non-root users, I can drop these users every test which should effectively
+ // reset the environment.
+
+ if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_1))
+ conn.securityOperations().dropLocalUser(AUDIT_USER_1);
+ if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
+ conn.securityOperations().dropLocalUser(AUDIT_USER_2);
+ if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
+ conn.securityOperations().dropLocalUser(THIRD_TEST_TABLE_NAME);
+ if (conn.tableOperations().exists(NEW_TEST_TABLE_NAME))
+ conn.tableOperations().delete(NEW_TEST_TABLE_NAME);
+ if (conn.tableOperations().exists(OLD_TEST_TABLE_NAME))
+ conn.tableOperations().delete(OLD_TEST_TABLE_NAME);
+
+ // This will set the lastAuditTimestamp for the first test
+ getAuditMessages("setup");
+
+ }
+
+ @Test
+ public void testTableOperationsAudits() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_TABLE);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Testing activity begins here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
+ auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
+ Map<String,String> emptyMap = Collections.emptyMap();
+ Set<String> emptySet = Collections.emptySet();
+ auditConnector.tableOperations().clone(NEW_TEST_TABLE_NAME, OLD_TEST_TABLE_NAME, true, emptyMap, emptySet);
+ auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
+ auditConnector.tableOperations().offline(NEW_TEST_TABLE_NAME);
+ auditConnector.tableOperations().delete(NEW_TEST_TABLE_NAME);
+ // Testing activity ends here
+
+ ArrayList<String> auditMessages = getAuditMessages("testTableOperationsAudits");
+
+ assertEquals(1, findAuditMessage(auditMessages, "action: createTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: renameTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: cloneTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: offlineTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
+
+ }
+
+ @Test
+ public void testUserOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, InterruptedException, IOException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_USER);
+ grantEverySystemPriv(conn, AUDIT_USER_1);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Start testing activities here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.securityOperations().createLocalUser(AUDIT_USER_2, new PasswordToken(PASSWORD));
+
+ // It seems only root can grant stuff.
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
+ conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
+ auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
+ conn.securityOperations().grantTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
+ conn.securityOperations().revokeTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
+ auditConnector.securityOperations().changeLocalUserPassword(AUDIT_USER_2, new PasswordToken("anything"));
+ auditConnector.securityOperations().changeUserAuthorizations(AUDIT_USER_2, auths);
+ auditConnector.securityOperations().dropLocalUser(AUDIT_USER_2);
+ // Stop testing activities here
+
+ ArrayList<String> auditMessages = getAuditMessages("testUserOperationsAudits");
+
+ assertEquals(1, findAuditMessage(auditMessages, "action: createUser; targetUser: " + AUDIT_USER_2).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: grantSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: revokeSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: grantTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: revokeTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: changePassword; targetUser: " + AUDIT_USER_2 + "").size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: changeAuthorizations; targetUser: " + AUDIT_USER_2 + "; authorizations: " + auths.toString())
+ .size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: dropUser; targetUser: " + AUDIT_USER_2).size());
+ }
+
+ @Test
+ public void testImportExportOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException,
+ IOException, InterruptedException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
+ grantEverySystemPriv(conn, AUDIT_USER_1);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Start testing activities here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
+
+ // Insert some play data
+ BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
+ Mutation m = new Mutation("myRow");
+ m.put("cf1", "cq1", "v1");
+ m.put("cf1", "cq2", "v3");
+ bw.addMutation(m);
+ bw.close();
+
+ // Prepare to export the table
+ File exportDir = new File(getCluster().getConfig().getDir().toString() + "/export");
+
+ auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
+ auditConnector.tableOperations().exportTable(OLD_TEST_TABLE_NAME, exportDir.toString());
+
+ // We've exported the table metadata to the MiniAccumuloCluster root dir. Grab the .rf file path to re-import it
+ File distCpTxt = new File(exportDir.toString() + "/distcp.txt");
+ File importFile = null;
- LineIterator it = FileUtils.lineIterator(distCpTxt, Constants.UTF8.name());
++ LineIterator it = FileUtils.lineIterator(distCpTxt, UTF_8.name());
+
+ // Just grab the first rf file, it will do for now.
+ String filePrefix = "file:";
+ try {
+ while (it.hasNext() && importFile == null) {
+ String line = it.nextLine();
+ if (line.matches(".*\\.rf")) {
+ importFile = new File(line.replaceFirst(filePrefix, ""));
+ }
+ }
+ } finally {
+ LineIterator.closeQuietly(it);
+ }
+ FileUtils.copyFileToDirectory(importFile, exportDir);
+ auditConnector.tableOperations().importTable(NEW_TEST_TABLE_NAME, exportDir.toString());
+
+ // Now do a Directory (bulk) import of the same data.
+ auditConnector.tableOperations().create(THIRD_TEST_TABLE_NAME);
+ File failDir = new File(exportDir + "/tmp");
+ failDir.mkdirs();
+ auditConnector.tableOperations().importDirectory(THIRD_TEST_TABLE_NAME, exportDir.toString(), failDir.toString(), false);
+ auditConnector.tableOperations().online(OLD_TEST_TABLE_NAME);
+
+ // Stop testing activities here
+
+ ArrayList<String> auditMessages = getAuditMessages("testImportExportOperationsAudits");
+
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME)).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
+ .size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_EXPORT_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, exportDir.toString())).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ String.format(AuditedSecurityOperation.CAN_IMPORT_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME, filePrefix + exportDir.toString())).size());
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME)).size());
+ assertEquals(
+ 1,
+ findAuditMessage(
+ auditMessages,
+ String.format(AuditedSecurityOperation.CAN_BULK_IMPORT_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME, filePrefix + exportDir.toString(), filePrefix
+ + failDir.toString())).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "onlineTable", OLD_TEST_TABLE_NAME))
+ .size());
+
+ }
+
+ @Test
+ public void testDataOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
+ grantEverySystemPriv(conn, AUDIT_USER_1);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Start testing activities here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
+
+ // Insert some play data
+ BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
+ Mutation m = new Mutation("myRow");
+ m.put("cf1", "cq1", "v1");
+ m.put("cf1", "cq2", "v3");
+ bw.addMutation(m);
+ bw.close();
+
+ // Start testing activities here
+ // A regular scan
+ Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
+ for (Map.Entry<Key,Value> entry : scanner) {
+ System.out.println("Scanner row: " + entry.getKey() + " " + entry.getValue());
+ }
+ scanner.close();
+
+ // A batch scan
+ BatchScanner bs = auditConnector.createBatchScanner(OLD_TEST_TABLE_NAME, auths, 1);
+ bs.fetchColumn(new Text("cf1"), new Text("cq1"));
+ bs.setRanges(Arrays.asList(new Range("myRow", "myRow~")));
+
+ for (Map.Entry<Key,Value> entry : bs) {
+ System.out.println("BatchScanner row: " + entry.getKey() + " " + entry.getValue());
+ }
+ bs.close();
+
+ // Delete some data.
+ auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
+
+ // End of testing activities
+
+ ArrayList<String> auditMessages = getAuditMessages("testDataOperationsAudits");
+ assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
+
+ }
+
+ @Test
+ public void testDeniedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ // Create our user with no privs
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.tableOperations().create(OLD_TEST_TABLE_NAME);
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+
+ // Start testing activities
+ // We should get denied or / failed audit messages here.
+ // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
+
+ try {
+ auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().clone(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME, true, Collections.<String,String> emptyMap(),
+ Collections.<String> emptySet());
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
+ scanner.iterator().next().getKey();
+ } catch (RuntimeException ex) {}
+ try {
+ auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
+ } catch (AccumuloSecurityException ex) {}
+
+ // ... that will do for now.
+ // End of testing activities
+
+ ArrayList<String> auditMessages = getAuditMessages("testDeniedAudits");
+ assertEquals(1,
+ findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME))
+ .size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_RENAME_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CLONE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME))
+ .size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
+ .size());
+ assertEquals(1, findAuditMessage(auditMessages, "operation: denied;.*" + "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
+ }
+
+ @Test
+ public void testFailedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ // Start testing activities
+ // Test that we get a few "failed" audit messages come through when we tell it to do dumb stuff
+ // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
+ try {
+ conn.securityOperations().dropLocalUser(AUDIT_USER_2);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ conn.securityOperations().createLocalUser("root", new PasswordToken("super secret"));
+ } catch (AccumuloSecurityException ex) {}
+ ArrayList<String> auditMessages = getAuditMessages("testFailedAudits");
+ // ... that will do for now.
+ // End of testing activities
+
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.DROP_USER_AUDIT_TEMPLATE, AUDIT_USER_2)).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ String.format(AuditedSecurityOperation.REVOKE_SYSTEM_PERMISSION_AUDIT_TEMPLATE, SystemPermission.ALTER_TABLE, AUDIT_USER_2)).size());
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CREATE_USER_AUDIT_TEMPLATE, "root", "")).size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index c1e779e,0000000..8188deb
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@@ -1,1248 -1,0 +1,1248 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.NamespaceExistsException;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.hadoop.io.Text;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class NamespacesIT extends SimpleMacIT {
+
+ private Connector c;
+ private String namespace;
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Before
+ public void setUpConnectorAndNamespace() throws Exception {
+ // prepare a unique namespace and get a new root connector for each test
+ c = getConnector();
+ namespace = "ns_" + getUniqueNames(1)[0];
+ }
+
+ @After
+ public void swingMjölnir() throws Exception {
+ // clean up any added tables, namespaces, and users, after each test
+ for (String t : c.tableOperations().list())
+ if (!Tables.qualify(t).getFirst().equals(Namespaces.ACCUMULO_NAMESPACE))
+ c.tableOperations().delete(t);
+ assertEquals(2, c.tableOperations().list().size());
+ for (String n : c.namespaceOperations().list())
+ if (!n.equals(Namespaces.ACCUMULO_NAMESPACE) && !n.equals(Namespaces.DEFAULT_NAMESPACE))
+ c.namespaceOperations().delete(n);
+ assertEquals(2, c.namespaceOperations().list().size());
+ for (String u : c.securityOperations().listLocalUsers())
+ if (!"root".equals(u))
+ c.securityOperations().dropLocalUser(u);
+ assertEquals(1, c.securityOperations().listLocalUsers().size());
+ }
+
+ @Test
+ public void checkReservedNamespaces() throws Exception {
+ assertEquals(c.namespaceOperations().defaultNamespace(), Namespaces.DEFAULT_NAMESPACE);
+ assertEquals(c.namespaceOperations().systemNamespace(), Namespaces.ACCUMULO_NAMESPACE);
+ }
+
+ @Test
+ public void checkBuiltInNamespaces() throws Exception {
+ assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
+ assertTrue(c.namespaceOperations().exists(Namespaces.ACCUMULO_NAMESPACE));
+ }
+
+ @Test
+ public void createTableInDefaultNamespace() throws Exception {
+ String tableName = "1";
+ c.tableOperations().create(tableName);
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ @Test(expected = AccumuloException.class)
+ public void createTableInAccumuloNamespace() throws Exception {
+ String tableName = Namespaces.ACCUMULO_NAMESPACE + ".1";
+ assertFalse(c.tableOperations().exists(tableName));
+ c.tableOperations().create(tableName); // should fail
+ }
+
+ @Test(expected = AccumuloSecurityException.class)
+ public void deleteDefaultNamespace() throws Exception {
+ c.namespaceOperations().delete(Namespaces.DEFAULT_NAMESPACE); // should fail
+ }
+
+ @Test(expected = AccumuloSecurityException.class)
+ public void deleteAccumuloNamespace() throws Exception {
+ c.namespaceOperations().delete(Namespaces.ACCUMULO_NAMESPACE); // should fail
+ }
+
+ @Test
+ public void createTableInMissingNamespace() throws Exception {
+ String t = namespace + ".1";
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t));
+ try {
+ c.tableOperations().create(t);
+ fail();
+ } catch (AccumuloException e) {
+ assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t));
+ }
+ }
+
+ @Test
+ public void createAndDeleteNamespace() throws Exception {
+ String t1 = namespace + ".1";
+ String t2 = namespace + ".2";
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ try {
+ c.namespaceOperations().delete(namespace);
+ } catch (NamespaceNotFoundException e) {}
+ try {
+ c.tableOperations().delete(t1);
+ } catch (TableNotFoundException e) {
+ assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
+ }
+ c.namespaceOperations().create(namespace);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.tableOperations().create(t1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.tableOperations().create(t2);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t2));
+ c.tableOperations().delete(t1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t2));
+ c.tableOperations().delete(t2);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.namespaceOperations().delete(namespace);
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ }
+
+ @Test(expected = NamespaceNotEmptyException.class)
+ public void deleteNonEmptyNamespace() throws Exception {
+ String tableName1 = namespace + ".1";
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(tableName1));
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(tableName1));
+ c.namespaceOperations().delete(namespace); // should fail
+ }
+
+ @Test
+ public void verifyPropertyInheritance() throws Exception {
+ String t0 = "0";
+ String t1 = namespace + ".1";
+ String t2 = namespace + ".2";
+
+ String k = Property.TABLE_SCAN_MAXMEM.getKey();
+ String v = "42K";
+
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ c.tableOperations().create(t0);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t0));
+
+ // verify no property
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(t1, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // set property and verify
+ c.namespaceOperations().setProperty(namespace, k, v);
+ assertTrue(checkNamespaceHasProp(namespace, k, v));
+ assertTrue(checkTableHasProp(t1, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // add a new table to namespace and verify
+ assertFalse(c.tableOperations().exists(t2));
+ c.tableOperations().create(t2);
+ assertTrue(c.tableOperations().exists(t2));
+ assertTrue(checkNamespaceHasProp(namespace, k, v));
+ assertTrue(checkTableHasProp(t1, k, v));
+ assertTrue(checkTableHasProp(t2, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // remove property and verify
+ c.namespaceOperations().removeProperty(namespace, k);
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(t1, k, v));
+ assertFalse(checkTableHasProp(t2, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // set property on default namespace and verify
+ c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, k, v);
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(t1, k, v));
+ assertFalse(checkTableHasProp(t2, k, v));
+ assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertTrue(checkTableHasProp(t0, k, v));
+
+ // test that table properties override namespace properties
+ String k2 = Property.TABLE_FILE_MAX.getKey();
+ String v2 = "42";
+ String table_v2 = "13";
+
+ // set new property on some
+ c.namespaceOperations().setProperty(namespace, k2, v2);
+ c.tableOperations().setProperty(t2, k2, table_v2);
+ assertTrue(checkNamespaceHasProp(namespace, k2, v2));
+ assertTrue(checkTableHasProp(t1, k2, v2));
+ assertTrue(checkTableHasProp(t2, k2, table_v2));
+
+ c.tableOperations().delete(t1);
+ c.tableOperations().delete(t2);
+ c.tableOperations().delete(t0);
+ c.namespaceOperations().delete(namespace);
+ }
+
+ @Test
+ public void verifyIteratorInheritance() throws Exception {
+ String t1 = namespace + ".1";
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ String iterName = namespace + "_iter";
+
+ BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes()));
+ bw.addMutation(m);
+ bw.flush();
+ bw.close();
+
+ IteratorSetting setting = new IteratorSetting(250, iterName, SimpleFilter.class.getName());
+
+ // verify can see inserted entry
+ Scanner s = c.createScanner(t1, Authorizations.EMPTY);
+ assertTrue(s.iterator().hasNext());
+ assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
+ assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+
+ // verify entry is filtered out (also, verify conflict checking API)
+ c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
+ c.namespaceOperations().attachIterator(namespace, setting);
+ UtilWaitThread.sleep(2*1000);
+ try {
+ c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
+ fail();
+ } catch (AccumuloException e) {
+ assertEquals(IllegalArgumentException.class.getName(), e.getCause().getClass().getName());
+ }
+ IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan);
+ assertEquals(setting, setting2);
+ assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
+ assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertFalse(s.iterator().hasNext());
+
+ // verify can see inserted entry again
+ c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
+ UtilWaitThread.sleep(2*1000);
+ assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
+ assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertTrue(s.iterator().hasNext());
+ }
+
+ @Test
+ public void cloneTable() throws Exception {
+ String namespace2 = namespace + "_clone";
+ String t1 = namespace + ".1";
+ String t2 = namespace + ".2";
+ String t3 = namespace2 + ".2";
+ String k1 = Property.TABLE_FILE_MAX.getKey();
+ String k2 = Property.TABLE_FILE_REPLICATION.getKey();
+ String k1v1 = "55";
+ String k1v2 = "66";
+ String k2v1 = "5";
+ String k2v2 = "6";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+
+ try {
+ // try to clone before namespace exists
+ c.tableOperations().clone(t1, t3, false, null, null); // should fail
+ fail();
+ } catch (AccumuloException e) {
+ assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
+ }
+
+ // try to clone before when target tables exist
+ c.namespaceOperations().create(namespace2);
+ c.tableOperations().create(t2);
+ c.tableOperations().create(t3);
+ for (String t : Arrays.asList(t2, t3)) {
+ try {
+ c.tableOperations().clone(t1, t, false, null, null); // should fail
+ fail();
+ } catch (TableExistsException e) {
+ c.tableOperations().delete(t);
+ }
+ }
+
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+
+ // set property with different values in two namespaces and a separate property with different values on the table and both namespaces
+ assertFalse(checkNamespaceHasProp(namespace, k1, k1v1));
+ assertFalse(checkNamespaceHasProp(namespace2, k1, k1v2));
+ assertFalse(checkTableHasProp(t1, k1, k1v1));
+ assertFalse(checkTableHasProp(t1, k1, k1v2));
+ assertFalse(checkNamespaceHasProp(namespace, k2, k2v1));
+ assertFalse(checkNamespaceHasProp(namespace2, k2, k2v1));
+ assertFalse(checkTableHasProp(t1, k2, k2v1));
+ assertFalse(checkTableHasProp(t1, k2, k2v2));
+ c.namespaceOperations().setProperty(namespace, k1, k1v1);
+ c.namespaceOperations().setProperty(namespace2, k1, k1v2);
+ c.namespaceOperations().setProperty(namespace, k2, k2v1);
+ c.namespaceOperations().setProperty(namespace2, k2, k2v1);
+ c.tableOperations().setProperty(t1, k2, k2v2);
+ assertTrue(checkNamespaceHasProp(namespace, k1, k1v1));
+ assertTrue(checkNamespaceHasProp(namespace2, k1, k1v2));
+ assertTrue(checkTableHasProp(t1, k1, k1v1));
+ assertFalse(checkTableHasProp(t1, k1, k1v2));
+ assertTrue(checkNamespaceHasProp(namespace, k2, k2v1));
+ assertTrue(checkNamespaceHasProp(namespace2, k2, k2v1));
+ assertFalse(checkTableHasProp(t1, k2, k2v1));
+ assertTrue(checkTableHasProp(t1, k2, k2v2));
+
+ // clone twice, once in same namespace, once in another
+ for (String t : Arrays.asList(t2, t3))
+ c.tableOperations().clone(t1, t, false, null, null);
+
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t2));
+ assertTrue(c.tableOperations().exists(t3));
+
+ // verify the properties got transferred
+ assertTrue(checkTableHasProp(t1, k1, k1v1));
+ assertTrue(checkTableHasProp(t2, k1, k1v1));
+ assertTrue(checkTableHasProp(t3, k1, k1v2));
+ assertTrue(checkTableHasProp(t1, k2, k2v2));
+ assertTrue(checkTableHasProp(t2, k2, k2v2));
+ assertTrue(checkTableHasProp(t3, k2, k2v2));
+ }
+
+ @Test
+ public void renameNamespaceWithTable() throws Exception {
+ String namespace2 = namespace + "_renamed";
+ String t1 = namespace + ".t";
+ String t2 = namespace2 + ".t";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t2));
+
+ String namespaceId = c.namespaceOperations().namespaceIdMap().get(namespace);
+ String tableId = c.tableOperations().tableIdMap().get(t1);
+
+ c.namespaceOperations().rename(namespace, namespace2);
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(c.tableOperations().exists(t2));
+
+ // verify id's didn't change
+ String namespaceId2 = c.namespaceOperations().namespaceIdMap().get(namespace2);
+ String tableId2 = c.tableOperations().tableIdMap().get(t2);
+
+ assertEquals(namespaceId, namespaceId2);
+ assertEquals(tableId, tableId2);
+ }
+
+ @Test
+ public void verifyConstraintInheritance() throws Exception {
+ String t1 = namespace + ".1";
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1, false);
+ String constraintClassName = NumericValueConstraint.class.getName();
+
+ assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
+ assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
+
+ c.namespaceOperations().addConstraint(namespace, constraintClassName);
+ assertTrue(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
+ assertTrue(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
+ int num = c.namespaceOperations().listConstraints(namespace).get(constraintClassName);
+ assertEquals(num, (int) c.tableOperations().listConstraints(t1).get(constraintClassName));
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+
+ Mutation m1 = new Mutation("r1");
+ Mutation m2 = new Mutation("r2");
+ Mutation m3 = new Mutation("r3");
- m1.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
- m2.put("e", "f", new Value("123".getBytes(Constants.UTF8)));
- m3.put("c", "d", new Value("zyxwv".getBytes(Constants.UTF8)));
++ m1.put("a", "b", new Value("abcde".getBytes(UTF_8)));
++ m2.put("e", "f", new Value("123".getBytes(UTF_8)));
++ m3.put("c", "d", new Value("zyxwv".getBytes(UTF_8)));
+ BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
+ bw.addMutations(Arrays.asList(m1, m2, m3));
+ try {
+ bw.close();
+ fail();
+ } catch (MutationsRejectedException e) {
+ assertEquals(1, e.getConstraintViolationSummaries().size());
+ assertEquals(2, e.getConstraintViolationSummaries().get(0).getNumberOfViolatingMutations());
+ }
+ c.namespaceOperations().removeConstraint(namespace, num);
+ assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
+ assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+
+ bw = c.createBatchWriter(t1, new BatchWriterConfig());
+ bw.addMutations(Arrays.asList(m1, m2, m3));
+ bw.close();
+ }
+
+ @Test
+ public void renameTable() throws Exception {
+ String namespace2 = namespace + "_renamed";
+ String t1 = namespace + ".1";
+ String t2 = namespace2 + ".2";
+ String t3 = namespace + ".3";
+ String t4 = namespace + ".4";
+ String t5 = "5";
+
+ c.namespaceOperations().create(namespace);
+ c.namespaceOperations().create(namespace2);
+
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+ assertFalse(c.tableOperations().exists(t4));
+ assertFalse(c.tableOperations().exists(t5));
+
+ c.tableOperations().create(t1);
+
+ try {
+ c.tableOperations().rename(t1, t2);
+ fail();
+ } catch (AccumuloException e) {
+ // this is expected, because we don't allow renames across namespaces
+ assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
+ assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
+ assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
+ }
+
+ try {
+ c.tableOperations().rename(t1, t5);
+ fail();
+ } catch (AccumuloException e) {
+ // this is expected, because we don't allow renames across namespaces
+ assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
+ assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
+ assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
+ }
+
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+ assertFalse(c.tableOperations().exists(t4));
+ assertFalse(c.tableOperations().exists(t5));
+
+ // fully qualified rename
+ c.tableOperations().rename(t1, t3);
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ assertTrue(c.tableOperations().exists(t3));
+ assertFalse(c.tableOperations().exists(t4));
+ assertFalse(c.tableOperations().exists(t5));
+ }
+
+ /**
+ * Tests new Namespace permissions as well as modifications to Table permissions because of namespaces. Checks each permission to first make sure the user
+ * doesn't have permission to perform the action, then root grants them the permission and we check to make sure they could perform the action.
+ */
+ @Test
+ public void testPermissions() throws Exception {
+ String u1 = "u1";
+ String u2 = "u2";
+ PasswordToken pass = new PasswordToken("pass");
+
+ String n1 = namespace;
+ String t1 = n1 + ".1";
+ String t2 = n1 + ".2";
+ String t3 = n1 + ".3";
+
+ String n2 = namespace + "_2";
+
+ c.namespaceOperations().create(n1);
+ c.tableOperations().create(t1);
+
+ c.securityOperations().createLocalUser(u1, pass);
+
+ Connector user1Con = c.getInstance().getConnector(u1, pass);
+
+ try {
+ user1Con.tableOperations().create(t2);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
+ user1Con.tableOperations().create(t2);
+ assertTrue(c.tableOperations().list().contains(t2));
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
+
+ try {
+ user1Con.tableOperations().delete(t1);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
+ user1Con.tableOperations().delete(t1);
+ assertTrue(!c.tableOperations().list().contains(t1));
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
+
+ c.tableOperations().create(t3);
+ BatchWriter bw = c.createBatchWriter(t3, null);
+ Mutation m = new Mutation("row");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ Iterator<Entry<Key,Value>> i = user1Con.createScanner(t3, new Authorizations()).iterator();
+ try {
+ i.next();
+ fail();
+ } catch (RuntimeException e) {
+ assertEquals(AccumuloSecurityException.class.getName(), e.getCause().getClass().getName());
+ expectPermissionDenied((AccumuloSecurityException) e.getCause());
+ }
+
+ m = new Mutation(u1);
+ m.put("cf", "cq", "turtles");
+ bw = user1Con.createBatchWriter(t3, null);
+ try {
+ bw.addMutation(m);
+ bw.close();
+ fail();
+ } catch (MutationsRejectedException e) {
+ assertEquals(1, e.getAuthorizationFailuresMap().size());
+ assertEquals(1, e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().size());
+ switch (e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().iterator().next()) {
+ case PERMISSION_DENIED:
+ break;
+ default:
+ fail();
+ }
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.READ);
+ i = user1Con.createScanner(t3, new Authorizations()).iterator();
+ assertTrue(i.hasNext());
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.READ);
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.WRITE);
+ m = new Mutation(u1);
+ m.put("cf", "cq", "turtles");
+ bw = user1Con.createBatchWriter(t3, null);
+ bw.addMutation(m);
+ bw.close();
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.WRITE);
+
+ try {
+ user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
+ user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
+ user1Con.tableOperations().removeProperty(t3, Property.TABLE_FILE_MAX.getKey());
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
+
+ try {
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "55");
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "42");
+ user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
+
+ c.securityOperations().createLocalUser(u2, pass);
+ try {
+ user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.GRANT);
+ user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
+ user1Con.securityOperations().revokeNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.GRANT);
+
+ try {
+ user1Con.namespaceOperations().create(n2);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
+ user1Con.namespaceOperations().create(n2);
+ c.securityOperations().revokeSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
+
+ c.securityOperations().revokeNamespacePermission(u1, n2, NamespacePermission.DROP_NAMESPACE);
+ try {
+ user1Con.namespaceOperations().delete(n2);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
+ user1Con.namespaceOperations().delete(n2);
+ c.securityOperations().revokeSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
+
+ try {
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
+ user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
+ c.securityOperations().revokeSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
+ }
+
+ @Test
+ public void verifySystemPropertyInheritance() throws Exception {
+ String t1 = "1";
+ String t2 = namespace + "." + t1;
+ c.tableOperations().create(t1);
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t2);
+
+ // verify iterator inheritance
+ _verifySystemPropertyInheritance(t1, t2, Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.sum", "20," + SimpleFilter.class.getName(), false);
+
+ // verify constraint inheritance
+ _verifySystemPropertyInheritance(t1, t2, Property.TABLE_CONSTRAINT_PREFIX.getKey() + "42", NumericValueConstraint.class.getName(), false);
+
+ // verify other inheritance
+ _verifySystemPropertyInheritance(t1, t2, Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "dummy", "dummy", true);
+ }
+
+ private void _verifySystemPropertyInheritance(String defaultNamespaceTable, String namespaceTable, String k, String v, boolean systemNamespaceShouldInherit)
+ throws Exception {
+ // nobody should have any of these properties yet
+ assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
+ assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(RootTable.NAME, k, v));
+ assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(namespaceTable, k, v));
+
+ // set the filter, verify that accumulo namespace is the only one unaffected
+ c.instanceOperations().setProperty(k, v);
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+ assertTrue(c.instanceOperations().getSystemConfiguration().containsValue(v));
+ assertEquals(systemNamespaceShouldInherit, checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
+ assertEquals(systemNamespaceShouldInherit, checkTableHasProp(RootTable.NAME, k, v));
+ assertEquals(systemNamespaceShouldInherit, checkTableHasProp(MetadataTable.NAME, k, v));
+ assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertTrue(checkTableHasProp(defaultNamespaceTable, k, v));
+ assertTrue(checkNamespaceHasProp(namespace, k, v));
+ assertTrue(checkTableHasProp(namespaceTable, k, v));
+
+ // verify it is no longer inherited
+ c.instanceOperations().removeProperty(k);
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+ assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
+ assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(RootTable.NAME, k, v));
+ assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(namespaceTable, k, v));
+ }
+
+ @Test
+ public void listNamespaces() throws Exception {
+ SortedSet<String> namespaces = c.namespaceOperations().list();
+ Map<String,String> map = c.namespaceOperations().namespaceIdMap();
+ assertEquals(2, namespaces.size());
+ assertEquals(2, map.size());
+ assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
+ assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
+ assertFalse(namespaces.contains(namespace));
+ assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
+ assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
+ assertNull(map.get(namespace));
+
+ c.namespaceOperations().create(namespace);
+ namespaces = c.namespaceOperations().list();
+ map = c.namespaceOperations().namespaceIdMap();
+ assertEquals(3, namespaces.size());
+ assertEquals(3, map.size());
+ assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
+ assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
+ assertTrue(namespaces.contains(namespace));
+ assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
+ assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
+ assertNotNull(map.get(namespace));
+
+ c.namespaceOperations().delete(namespace);
+ namespaces = c.namespaceOperations().list();
+ map = c.namespaceOperations().namespaceIdMap();
+ assertEquals(2, namespaces.size());
+ assertEquals(2, map.size());
+ assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
+ assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
+ assertFalse(namespaces.contains(namespace));
+ assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
+ assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
+ assertNull(map.get(namespace));
+ }
+
+ @Test
+ public void loadClass() throws Exception {
+ assertTrue(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
+ assertFalse(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, "dummy", SortedKeyValueIterator.class.getName()));
+ try {
+ c.namespaceOperations().testClassLoad(namespace, "dummy", "dummy");
+ fail();
+ } catch (NamespaceNotFoundException e) {}
+ }
+
+ @Test
+ public void testModifyingPermissions() throws Exception {
+ String tableName = namespace + ".modify";
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName);
+ assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
+ c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
+ assertFalse(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
+ c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
+ assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
+ c.tableOperations().delete(tableName);
+
+ try {
+ c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
+ c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ assertFalse(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
+ c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
+
+ c.namespaceOperations().delete(namespace);
+
+ try {
+ c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ }
+
+ @Test
+ public void verifyTableOperationsExceptions() throws Exception {
+ String tableName = namespace + ".1";
+ IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
+ Text a = new Text("a");
+ Text z = new Text("z");
+ TableOperations ops = c.tableOperations();
+
+ // this one doesn't throw an exception, so don't fail; just check that it works
+ assertFalse(ops.exists(tableName));
+
+ // table operations that should throw an AccumuloException caused by NamespaceNotFoundException
+ int numRun = 0;
+ ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.create(tableName);
+ fail();
+ case 1:
+ ops.create("a");
+ ops.clone("a", tableName, true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
+ fail();
+ case 2:
+ ops.importTable(tableName, System.getProperty("user.dir") + "/target");
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(3, i); // check test integrity
+ assertEquals(3, numRun); // check test integrity
+ break ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof AccumuloException) || !(e.getCause() instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+
+ // table operations that should throw an AccumuloException caused by a TableNotFoundException caused by a NamespaceNotFoundException
+ // these are here because we didn't declare TableNotFoundException in the API :(
+ numRun = 0;
+ ACCUMULOEXCEPTIONS_TABLENOTFOUND: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.removeConstraint(tableName, 0);
+ fail();
+ case 1:
+ ops.removeProperty(tableName, "a");
+ fail();
+ case 2:
+ ops.setProperty(tableName, "a", "b");
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(3, i); // check test integrity
+ assertEquals(3, numRun); // check test integrity
+ break ACCUMULOEXCEPTIONS_TABLENOTFOUND;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof AccumuloException) || !(e.getCause() instanceof TableNotFoundException)
+ || !(e.getCause().getCause() instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+
+ // table operations that should throw a TableNotFoundException caused by NamespaceNotFoundException
+ numRun = 0;
+ TABLENOTFOUNDEXCEPTIONS: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.addConstraint(tableName, NumericValueConstraint.class.getName());
+ fail();
+ case 1:
+ ops.addSplits(tableName, new TreeSet<Text>());
+ fail();
+ case 2:
+ ops.attachIterator(tableName, setting);
+ fail();
+ case 3:
+ ops.cancelCompaction(tableName);
+ fail();
+ case 4:
+ ops.checkIteratorConflicts(tableName, setting, EnumSet.allOf(IteratorScope.class));
+ fail();
+ case 5:
+ ops.clearLocatorCache(tableName);
+ fail();
+ case 6:
+ ops.clone(tableName, "2", true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
+ fail();
+ case 7:
+ ops.compact(tableName, a, z, true, true);
+ fail();
+ case 8:
+ ops.delete(tableName);
+ fail();
+ case 9:
+ ops.deleteRows(tableName, a, z);
+ fail();
+ case 10:
+ ops.splitRangeByTablets(tableName, new Range(), 10);
+ fail();
+ case 11:
+ ops.exportTable(tableName, namespace + "_dir");
+ fail();
+ case 12:
+ ops.flush(tableName, a, z, true);
+ fail();
+ case 13:
+ ops.getDiskUsage(Collections.singleton(tableName));
+ fail();
+ case 14:
+ ops.getIteratorSetting(tableName, "a", IteratorScope.scan);
+ fail();
+ case 15:
+ ops.getLocalityGroups(tableName);
+ fail();
+ case 16:
+ ops.getMaxRow(tableName, Authorizations.EMPTY, a, true, z, true);
+ fail();
+ case 17:
+ ops.getProperties(tableName);
+ fail();
+ case 18:
+ ops.importDirectory(tableName, "", "", false);
+ fail();
+ case 19:
+ ops.testClassLoad(tableName, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail();
+ case 20:
+ ops.listConstraints(tableName);
+ fail();
+ case 21:
+ ops.listIterators(tableName);
+ fail();
+ case 22:
+ ops.listSplits(tableName);
+ fail();
+ case 23:
+ ops.merge(tableName, a, z);
+ fail();
+ case 24:
+ ops.offline(tableName, true);
+ fail();
+ case 25:
+ ops.online(tableName, true);
+ fail();
+ case 26:
+ ops.removeIterator(tableName, "a", EnumSet.of(IteratorScope.scan));
+ fail();
+ case 27:
+ ops.rename(tableName, tableName + "2");
+ fail();
+ case 28:
+ ops.setLocalityGroups(tableName, Collections.<String,Set<Text>> emptyMap());
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(29, i); // check test integrity
+ assertEquals(29, numRun); // check test integrity
+ break TABLENOTFOUNDEXCEPTIONS;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof TableNotFoundException) || !(e.getCause() instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+ }
+
+ @Test
+ public void verifyNamespaceOperationsExceptions() throws Exception {
+ IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
+ NamespaceOperations ops = c.namespaceOperations();
+
+ // this one doesn't throw an exception, so don't fail; just check that it works
+ assertFalse(ops.exists(namespace));
+
+ // namespace operations that should throw a NamespaceNotFoundException
+ int numRun = 0;
+ NAMESPACENOTFOUND: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.addConstraint(namespace, NumericValueConstraint.class.getName());
+ fail();
+ case 1:
+ ops.attachIterator(namespace, setting);
+ fail();
+ case 2:
+ ops.checkIteratorConflicts(namespace, setting, EnumSet.of(IteratorScope.scan));
+ fail();
+ case 3:
+ ops.delete(namespace);
+ fail();
+ case 4:
+ ops.getIteratorSetting(namespace, "thing", IteratorScope.scan);
+ fail();
+ case 5:
+ ops.getProperties(namespace);
+ fail();
+ case 6:
+ ops.listConstraints(namespace);
+ fail();
+ case 7:
+ ops.listIterators(namespace);
+ fail();
+ case 8:
+ ops.removeConstraint(namespace, 1);
+ fail();
+ case 9:
+ ops.removeIterator(namespace, "thing", EnumSet.allOf(IteratorScope.class));
+ fail();
+ case 10:
+ ops.removeProperty(namespace, "a");
+ fail();
+ case 11:
+ ops.rename(namespace, namespace + "2");
+ fail();
+ case 12:
+ ops.setProperty(namespace, "k", "v");
+ fail();
+ case 13:
+ ops.testClassLoad(namespace, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(14, i); // check test integrity
+ assertEquals(14, numRun); // check test integrity
+ break NAMESPACENOTFOUND;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+
+ // namespace operations that should throw a NamespaceExistsException
+ numRun = 0;
+ NAMESPACEEXISTS: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.create(namespace + "0");
+ ops.create(namespace + "0"); // should fail here
+ fail();
+ case 1:
+ ops.create(namespace + i + "_1");
+ ops.create(namespace + i + "_2");
+ ops.rename(namespace + i + "_1", namespace + i + "_2"); // should fail here
+ fail();
+ case 2:
+ ops.create(Namespaces.DEFAULT_NAMESPACE);
+ fail();
+ case 3:
+ ops.create(Namespaces.ACCUMULO_NAMESPACE);
+ fail();
+ case 4:
+ ops.create(namespace + i + "_1");
+ ops.rename(namespace + i + "_1", Namespaces.DEFAULT_NAMESPACE); // should fail here
+ fail();
+ case 5:
+ ops.create(namespace + i + "_1");
+ ops.rename(namespace + i + "_1", Namespaces.ACCUMULO_NAMESPACE); // should fail here
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(6, i); // check test integrity
+ assertEquals(6, numRun); // check test integrity
+ break NAMESPACEEXISTS;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof NamespaceExistsException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+ }
+
+ private boolean checkTableHasProp(String t, String propKey, String propVal) {
+ return checkHasProperty(t, propKey, propVal, true);
+ }
+
+ private boolean checkNamespaceHasProp(String n, String propKey, String propVal) {
+ return checkHasProperty(n, propKey, propVal, false);
+ }
+
+ private boolean checkHasProperty(String name, String propKey, String propVal, boolean nameIsTable) {
+ try {
+ Iterable<Entry<String,String>> iterable = nameIsTable ? c.tableOperations().getProperties(name) : c.namespaceOperations().getProperties(name);
+ for (Entry<String,String> e : iterable)
+ if (propKey.equals(e.getKey()))
+ return propVal.equals(e.getValue());
+ return false;
+ } catch (Exception e) {
+ fail();
+ return false;
+ }
+ }
+
+ public static class SimpleFilter extends Filter {
+ @Override
+ public boolean accept(Key k, Value v) {
+ if (k.getColumnFamily().toString().equals("a"))
+ return false;
+ return true;
+ }
+ }
+
+ private void expectPermissionDenied(AccumuloSecurityException sec) {
+ assertEquals(sec.getSecurityErrorCode().getClass(), SecurityErrorCode.class);
+ switch (sec.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ break;
+ default:
+ fail();
+ }
+ }
+
+}
[49/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Encoding.java
index 761fc1f,b26b363..259f783
--- a/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.util.Base64;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
index 7b84dc8,d1994ce..e103ac6
--- a/core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/FastFormat.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
public class FastFormat {
// this 7 to 8 times faster than String.format("%s%06d",prefix, num)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java
index 167933f,05067b5..c156311
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.core.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
index 90268ef,e71a377..4a0f9ef
--- a/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
index a57839b,617c079..66ad8f5
--- a/core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/TextUtil.java
@@@ -16,9 -16,10 +16,9 @@@
*/
package org.apache.accumulo.core.util;
- import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ import java.nio.ByteBuffer;
-
import org.apache.accumulo.core.Constants;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index 8f0812c,c276056..a386c04
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.client;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
@@@ -161,14 -155,8 +161,14 @@@ public class BatchWriterConfigTest
bwConfig.setMaxWriteThreads(24);
bwConfig.setTimeout(3, TimeUnit.SECONDS);
bytes = createBytes(bwConfig);
- assertEquals(" v#maxWriteThreads=24,timeout=3000", new String(bytes, StandardCharsets.UTF_8));
+ assertEquals(" v#maxWriteThreads=24,timeout=3000", new String(bytes, UTF_8));
checkBytes(bwConfig, bytes);
+
+ // test human-readable durability
+ bwConfig = new BatchWriterConfig();
+ bwConfig.setDurability(Durability.LOG);
+ bytes = createBytes(bwConfig);
- assertEquals(" e#durability=LOG", new String(bytes, StandardCharsets.UTF_8));
++ assertEquals(" e#durability=LOG", new String(bytes, UTF_8));
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
index 98de0b5,dce742f..5d0b604
--- a/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.client;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index 1e86f73,e2661e2..b45054a
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -17,6 -17,7 +17,7 @@@
package org.apache.accumulo.core.client.mock;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
index 46607ea,7bb3fa4..bbed50c
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
@@@ -16,11 -16,12 +16,11 @@@
*/
package org.apache.accumulo.core.client.security.tokens;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
import java.io.IOException;
import java.net.URL;
- import java.nio.charset.StandardCharsets;
--
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
index a3ce8d4,878b5e0..3751f02
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/PasswordTokenTest.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.client.security.tokens;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import javax.security.auth.DestroyFailedException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
index 8672836,d9a47d7..c76275b
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 591baaa,908c391..4326227
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
index cd26743,8022c14..0aa7649
--- a/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ValueTest.java
@@@ -17,6 -17,8 +17,8 @@@
*/
package org.apache.accumulo.core.data;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
index ec516d0,d8179df..2649f90
--- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.iterators.user;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.TreeMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
index cf91cb0,2577e48..1a46004
--- a/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.security.ColumnVisibility.quote;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
index 5ecfdfb,a4e9f02..c5746bc
--- a/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/CredentialsTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
index d8bff67,523279a..c9aa5d2
--- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index d80124c,9078ef9..0774849
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.security.crypto;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 8f46c89,eef06c3..e8879a5
--- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.core.util.format;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 8884cd9,af31cf2..60ff4b4
--- a/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.fate;
- import java.nio.charset.Charset;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@@ -38,8 -38,6 +39,7 @@@ import org.apache.zookeeper.KeeperExcep
* A utility to administer FATE operations
*/
public class AdminUtil<T> {
- private static final Charset UTF8 = Charset.forName("UTF-8");
+ private static final Logger log = Logger.getLogger(AdminUtil.class);
private boolean exitOnError = false;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
index 5fc1858,34e0235..0dc156e
--- a/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.fate;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
index ea7f02c,b2e263a..d10fcd2
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/DistributedReadWriteLock.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.fate.zookeeper;
- import java.nio.charset.Charset;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
index 74c525e,4907bc6..ee53ddd
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.fate.zookeeper;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
index 5fef905,f61c214..1391d98
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.fate.zookeeper;
- import java.nio.charset.Charset;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@@ -33,8 -34,7 +34,7 @@@ import org.apache.zookeeper.ZooKeeper
import org.apache.zookeeper.data.Stat;
public class ZooLock implements Watcher {
- private static final Charset UTF8 = Charset.forName("UTF-8");
- protected static final Logger log = Logger.getLogger(ZooLock.class);
+ private static final Logger log = Logger.getLogger(ZooLock.class);
public static final String LOCK_PREFIX = "zlock-";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
index ea7f4b9,2e2acc3..f1ba428
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReservation.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.fate.zookeeper;
- import java.nio.charset.Charset;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
----------------------------------------------------------------------
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index b6890c6,ff35364..0059af7
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.fate.zookeeper;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.net.UnknownHostException;
- import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index b591a68,0000000..29cf95d
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -1,522 -1,0 +1,523 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
- import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase.TokenSource;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.log4j.Level;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
+ */
+public class RangeInputSplit extends InputSplit implements Writable {
+ private Range range;
+ private String[] locations;
+ private String tableId, tableName, instanceName, zooKeepers, principal;
+ private TokenSource tokenSource;
+ private String tokenFile;
+ private AuthenticationToken token;
+ private Boolean offline, mockInstance, isolatedScan, localIterators;
+ private Authorizations auths;
+ private Set<Pair<Text,Text>> fetchedColumns;
+ private List<IteratorSetting> iterators;
+ private Level level;
+
+ public RangeInputSplit() {
+ range = new Range();
+ locations = new String[0];
+ tableName = "";
+ tableId = "";
+ }
+
+ public RangeInputSplit(RangeInputSplit split) throws IOException {
+ this.setRange(split.getRange());
+ this.setLocations(split.getLocations());
+ this.setTableName(split.getTableName());
+ this.setTableId(split.getTableId());
+ }
+
+ protected RangeInputSplit(String table, String tableId, Range range, String[] locations) {
+ this.range = range;
+ setLocations(locations);
+ this.tableName = table;
+ this.tableId = tableId;
+ }
+
+ public Range getRange() {
+ return range;
+ }
+
+ private static byte[] extractBytes(ByteSequence seq, int numBytes) {
+ byte[] bytes = new byte[numBytes + 1];
+ bytes[0] = 0;
+ for (int i = 0; i < numBytes; i++) {
+ if (i >= seq.length())
+ bytes[i + 1] = 0;
+ else
+ bytes[i + 1] = seq.byteAt(i);
+ }
+ return bytes;
+ }
+
+ public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
+ int maxDepth = Math.min(Math.max(end.length(), start.length()), position.length());
+ BigInteger startBI = new BigInteger(extractBytes(start, maxDepth));
+ BigInteger endBI = new BigInteger(extractBytes(end, maxDepth));
+ BigInteger positionBI = new BigInteger(extractBytes(position, maxDepth));
+ return (float) (positionBI.subtract(startBI).doubleValue() / endBI.subtract(startBI).doubleValue());
+ }
+
+ public float getProgress(Key currentKey) {
+ if (currentKey == null)
+ return 0f;
+ if (range.getStartKey() != null && range.getEndKey() != null) {
+ if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
+ // just look at the row progress
+ return getProgress(range.getStartKey().getRowData(), range.getEndKey().getRowData(), currentKey.getRowData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM) != 0) {
+ // just look at the column family progress
+ return getProgress(range.getStartKey().getColumnFamilyData(), range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM_COLQUAL) != 0) {
+ // just look at the column qualifier progress
+ return getProgress(range.getStartKey().getColumnQualifierData(), range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
+ }
+ }
+ // if we can't figure it out, then claim no progress
+ return 0f;
+ }
+
+ /**
+ * This implementation of length is only an estimate, it does not provide exact values. Do not have your code rely on this return value.
+ */
+ @Override
+ public long getLength() throws IOException {
+ Text startRow = range.isInfiniteStartKey() ? new Text(new byte[] {Byte.MIN_VALUE}) : range.getStartKey().getRow();
+ Text stopRow = range.isInfiniteStopKey() ? new Text(new byte[] {Byte.MAX_VALUE}) : range.getEndKey().getRow();
+ int maxCommon = Math.min(7, Math.min(startRow.getLength(), stopRow.getLength()));
+ long diff = 0;
+
+ byte[] start = startRow.getBytes();
+ byte[] stop = stopRow.getBytes();
+ for (int i = 0; i < maxCommon; ++i) {
+ diff |= 0xff & (start[i] ^ stop[i]);
+ diff <<= Byte.SIZE;
+ }
+
+ if (startRow.getLength() != stopRow.getLength())
+ diff |= 0xff;
+
+ return diff + 1;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException {
+ return Arrays.copyOf(locations, locations.length);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ range.readFields(in);
+ tableName = in.readUTF();
+ tableId = in.readUTF();
+ int numLocs = in.readInt();
+ locations = new String[numLocs];
+ for (int i = 0; i < numLocs; ++i)
+ locations[i] = in.readUTF();
+
+ if (in.readBoolean()) {
+ isolatedScan = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ offline = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ localIterators = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ mockInstance = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ int numColumns = in.readInt();
+ List<String> columns = new ArrayList<String>(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+ columns.add(in.readUTF());
+ }
+
+ fetchedColumns = InputConfigurator.deserializeFetchedColumns(columns);
+ }
+
+ if (in.readBoolean()) {
+ String strAuths = in.readUTF();
- auths = new Authorizations(strAuths.getBytes(StandardCharsets.UTF_8));
++ auths = new Authorizations(strAuths.getBytes(UTF_8));
+ }
+
+ if (in.readBoolean()) {
+ principal = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ int ordinal = in.readInt();
+ this.tokenSource = TokenSource.values()[ordinal];
+
+ switch (this.tokenSource) {
+ case INLINE:
+ String tokenClass = in.readUTF();
- byte[] base64TokenBytes = in.readUTF().getBytes(StandardCharsets.UTF_8);
++ byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
+ byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
+
+ this.token = AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes);
+ break;
+
+ case FILE:
+ this.tokenFile = in.readUTF();
+
+ break;
+ default:
+ throw new IOException("Cannot parse unknown TokenSource ordinal");
+ }
+ }
+
+ if (in.readBoolean()) {
+ instanceName = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ zooKeepers = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ int numIterators = in.readInt();
+ iterators = new ArrayList<IteratorSetting>(numIterators);
+ for (int i = 0; i < numIterators; i++) {
+ iterators.add(new IteratorSetting(in));
+ }
+ }
+
+ if (in.readBoolean()) {
+ level = Level.toLevel(in.readInt());
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ range.write(out);
+ out.writeUTF(tableName);
+ out.writeUTF(tableId);
+ out.writeInt(locations.length);
+ for (int i = 0; i < locations.length; ++i)
+ out.writeUTF(locations[i]);
+
+ out.writeBoolean(null != isolatedScan);
+ if (null != isolatedScan) {
+ out.writeBoolean(isolatedScan);
+ }
+
+ out.writeBoolean(null != offline);
+ if (null != offline) {
+ out.writeBoolean(offline);
+ }
+
+ out.writeBoolean(null != localIterators);
+ if (null != localIterators) {
+ out.writeBoolean(localIterators);
+ }
+
+ out.writeBoolean(null != mockInstance);
+ if (null != mockInstance) {
+ out.writeBoolean(mockInstance);
+ }
+
+ out.writeBoolean(null != fetchedColumns);
+ if (null != fetchedColumns) {
+ String[] cols = InputConfigurator.serializeColumns(fetchedColumns);
+ out.writeInt(cols.length);
+ for (String col : cols) {
+ out.writeUTF(col);
+ }
+ }
+
+ out.writeBoolean(null != auths);
+ if (null != auths) {
+ out.writeUTF(auths.serialize());
+ }
+
+ out.writeBoolean(null != principal);
+ if (null != principal) {
+ out.writeUTF(principal);
+ }
+
+ out.writeBoolean(null != tokenSource);
+ if (null != tokenSource) {
+ out.writeInt(tokenSource.ordinal());
+
+ if (null != token && null != tokenFile) {
+ throw new IOException("Cannot use both inline AuthenticationToken and file-based AuthenticationToken");
+ } else if (null != token) {
+ out.writeUTF(token.getClass().getCanonicalName());
+ out.writeUTF(Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ } else {
+ out.writeUTF(tokenFile);
+ }
+ }
+
+ out.writeBoolean(null != instanceName);
+ if (null != instanceName) {
+ out.writeUTF(instanceName);
+ }
+
+ out.writeBoolean(null != zooKeepers);
+ if (null != zooKeepers) {
+ out.writeUTF(zooKeepers);
+ }
+
+ out.writeBoolean(null != iterators);
+ if (null != iterators) {
+ out.writeInt(iterators.size());
+ for (IteratorSetting iterator : iterators) {
+ iterator.write(out);
+ }
+ }
+
+ out.writeBoolean(null != level);
+ if (null != level) {
+ out.writeInt(level.toInt());
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ sb.append("Range: ").append(range);
+ sb.append(" Locations: ").append(Arrays.asList(locations));
+ sb.append(" Table: ").append(tableName);
+ sb.append(" TableID: ").append(tableId);
+ sb.append(" InstanceName: ").append(instanceName);
+ sb.append(" zooKeepers: ").append(zooKeepers);
+ sb.append(" principal: ").append(principal);
+ sb.append(" tokenSource: ").append(tokenSource);
+ sb.append(" authenticationToken: ").append(token);
+ sb.append(" authenticationTokenFile: ").append(tokenFile);
+ sb.append(" Authorizations: ").append(auths);
+ sb.append(" offlineScan: ").append(offline);
+ sb.append(" mockInstance: ").append(mockInstance);
+ sb.append(" isolatedScan: ").append(isolatedScan);
+ sb.append(" localIterators: ").append(localIterators);
+ sb.append(" fetchColumns: ").append(fetchedColumns);
+ sb.append(" iterators: ").append(iterators);
+ sb.append(" logLevel: ").append(level);
+ return sb.toString();
+ }
+
+ /**
+ * Use {@link #getTableName}
+ */
+ @Deprecated
+ public String getTable() {
+ return getTableName();
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * Use {@link #setTableName}
+ */
+ @Deprecated
+ public void setTable(String table) {
+ setTableName(table);
+ }
+
+ public void setTableName(String table) {
+ this.tableName = table;
+ }
+
+ public void setTableId(String tableId) {
+ this.tableId = tableId;
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+
+ public Instance getInstance() {
+ if (null == instanceName) {
+ return null;
+ }
+
+ if (isMockInstance()) {
+ return new MockInstance(getInstanceName());
+ }
+
+ if (null == zooKeepers) {
+ return null;
+ }
+
+ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(getInstanceName()).withZkHosts(getZooKeepers()));
+ }
+
+ public String getInstanceName() {
+ return instanceName;
+ }
+
+ public void setInstanceName(String instanceName) {
+ this.instanceName = instanceName;
+ }
+
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ public void setZooKeepers(String zooKeepers) {
+ this.zooKeepers = zooKeepers;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ public void setToken(AuthenticationToken token) {
+ this.tokenSource = TokenSource.INLINE;
+ this.token = token;
+ }
+
+ public void setToken(String tokenFile) {
+ this.tokenSource = TokenSource.FILE;
+ this.tokenFile = tokenFile;
+ }
+
+ public Boolean isOffline() {
+ return offline;
+ }
+
+ public void setOffline(Boolean offline) {
+ this.offline = offline;
+ }
+
+ public void setLocations(String[] locations) {
+ this.locations = Arrays.copyOf(locations, locations.length);
+ }
+
+ public Boolean isMockInstance() {
+ return mockInstance;
+ }
+
+ public void setMockInstance(Boolean mockInstance) {
+ this.mockInstance = mockInstance;
+ }
+
+ public Boolean isIsolatedScan() {
+ return isolatedScan;
+ }
+
+ public void setIsolatedScan(Boolean isolatedScan) {
+ this.isolatedScan = isolatedScan;
+ }
+
+ public Authorizations getAuths() {
+ return auths;
+ }
+
+ public void setAuths(Authorizations auths) {
+ this.auths = auths;
+ }
+
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ public Boolean usesLocalIterators() {
+ return localIterators;
+ }
+
+ public void setUsesLocalIterators(Boolean localIterators) {
+ this.localIterators = localIterators;
+ }
+
+ public Set<Pair<Text,Text>> getFetchedColumns() {
+ return fetchedColumns;
+ }
+
+ public void setFetchedColumns(Collection<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = new HashSet<Pair<Text,Text>>();
+ for (Pair<Text,Text> columns : fetchedColumns) {
+ this.fetchedColumns.add(columns);
+ }
+ }
+
+ public void setFetchedColumns(Set<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = fetchedColumns;
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+
+ public void setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ }
+
+ public Level getLogLevel() {
+ return level;
+ }
+
+ public void setLogLevel(Level level) {
+ this.level = level;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index 2d9fc04,0000000..ae1d46f
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,405 -1,0 +1,404 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
- import java.nio.charset.StandardCharsets;
-
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.6.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN,
+ }
+
+ public static enum TokenSource {
+ FILE, INLINE;
+
+ private String prefix;
+
+ private TokenSource() {
+ prefix = name().toLowerCase() + ":";
+ }
+
+ public String prefix() {
+ return prefix;
+ }
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.6.0
+ */
+ public static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.6.0
+ */
+ public static enum GeneralOpts {
+ LOG_LEVEL,
+ VISIBILITY_CACHE_SIZE
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.6.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ checkArgument(principal != null, "principal is null");
+ checkArgument(token != null, "token is null");
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
+ TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
+ * Configuration. Token file created with "bin/accumulo create-token".
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param tokenFile
+ * the path to the token file in DFS
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ checkArgument(principal != null, "principal is null");
+ checkArgument(tokenFile != null, "tokenFile is null");
+
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
+ }
+
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ * @see #setConnectorInfo(Class, Configuration, String, String)
+ */
+ public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
+ String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
+ if (token == null || token.isEmpty())
+ return null;
+ if (token.startsWith(TokenSource.INLINE.prefix())) {
+ String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
+ if (args.length == 2)
- return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(StandardCharsets.UTF_8)));
++ return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(UTF_8)));
+ } else if (token.startsWith(TokenSource.FILE.prefix())) {
+ String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
+ return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
+ }
+
+ throw new IllegalStateException("Token was not properly serialized into the configuration");
+ }
+
+ /**
+ * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
+ *
+ * @param conf
+ * the Hadoop context for the configured job
+ * @return path to the token file as a String
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
+ FSDataInputStream in = null;
+ try {
+ URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
+ Path path = null;
+ for (URI u : uris) {
+ if (u.toString().equals(tokenFile)) {
+ path = new Path(u);
+ }
+ }
+ if (path == null) {
+ throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
+ }
+ FileSystem fs = FileSystem.get(conf);
+ in = fs.open(path);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
+ }
+ try (java.util.Scanner fileScanner = new java.util.Scanner(in)) {
+ while (fileScanner.hasNextLine()) {
+ Credentials creds = Credentials.deserialize(fileScanner.nextLine());
+ if (principal.equals(creds.getPrincipal())) {
+ return creds.getToken();
+ }
+ }
+ throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
+ }
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+ if (clientConfig != null) {
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
+ }
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.6.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ checkArgument(instanceName != null, "instanceName is null");
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.6.0
+ * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+ if (clientConfigString == null) {
+ String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+ String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
+ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
+ } else {
+ return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
+ }
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.6.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ checkArgument(level != null, "level is null");
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.6.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
+}
[24/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
index e44d8df,0000000..49b5d70
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AuditMessageIT.java
@@@ -1,496 -1,0 +1,496 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests that Accumulo is outputting audit messages as expected. Since this is using MiniAccumuloCluster, it could take a while if we test everything in
+ * isolation. We test blocks of related operations, run the whole test in one MiniAccumulo instance, trying to clean up objects between each test. The
+ * MiniAccumuloClusterTest sets up the log4j stuff differently to an installed instance, instead piping everything through stdout and writing to a set location
+ * so we have to find the logs and grep the bits we need out.
+ */
+public class AuditMessageIT extends ConfigurableMacIT {
+
+ private static final String AUDIT_USER_1 = "AuditUser1";
+ private static final String AUDIT_USER_2 = "AuditUser2";
+ private static final String PASSWORD = "password";
+ private static final String OLD_TEST_TABLE_NAME = "apples";
+ private static final String NEW_TEST_TABLE_NAME = "oranges";
+ private static final String THIRD_TEST_TABLE_NAME = "pears";
+ private static final Authorizations auths = new Authorizations("private", "public");
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {
+ new File(cfg.getConfDir(), "auditLog.xml").delete();
+ }
+
+ // Must be static to survive Junit re-initialising the class every time.
+ private static String lastAuditTimestamp;
+ private Connector auditConnector;
+ private Connector conn;
+
+ private static ArrayList<String> findAuditMessage(ArrayList<String> input, String pattern) {
+ ArrayList<String> result = new ArrayList<String>();
+ for (String s : input) {
+ if (s.matches(".*" + pattern + ".*"))
+ result.add(s);
+ }
+ return result;
+ }
+
+ /**
+ * Returns a List of Audit messages that have been grep'd out of the MiniAccumuloCluster output.
+ *
+ * @param stepName
+ * A unique name for the test being executed, to identify the System.out messages.
+ * @return A List of the Audit messages, sorted (so in chronological order).
+ */
+ private ArrayList<String> getAuditMessages(String stepName) throws IOException {
+ // ACCUMULO-3144 Make sure we give the processes enough time to flush the write buffer
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted waiting for data to be flushed to output streams");
+ }
+
+ for (MiniAccumuloClusterImpl.LogWriter lw : getCluster().getLogWriters()) {
+ lw.flush();
+ }
+
+ // Grab the audit messages
+ System.out.println("Start of captured audit messages for step " + stepName);
+
+ ArrayList<String> result = new ArrayList<String>();
+ for (File file : getCluster().getConfig().getLogDir().listFiles()) {
+ // We want to grab the files called .out
+ if (file.getName().contains(".out") && file.isFile() && file.canRead()) {
- LineIterator it = FileUtils.lineIterator(file, Constants.UTF8.name());
++ LineIterator it = FileUtils.lineIterator(file, UTF_8.name());
+ try {
+ while (it.hasNext()) {
+ String line = it.nextLine();
+ if (line.matches(".* \\[" + AuditedSecurityOperation.AUDITLOG + "\\s*\\].*")) {
+ // Only include the message if startTimestamp is null. or the message occurred after the startTimestamp value
+ if ((lastAuditTimestamp == null) || (line.substring(0, 23).compareTo(lastAuditTimestamp) > 0))
+ result.add(line);
+ }
+ }
+ } finally {
+ LineIterator.closeQuietly(it);
+ }
+ }
+ }
+ Collections.sort(result);
+
+ for (String s : result) {
+ System.out.println(s);
+ }
+ System.out.println("End of captured audit messages for step " + stepName);
+ if (result.size() > 0)
+ lastAuditTimestamp = (result.get(result.size() - 1)).substring(0, 23);
+
+ return result;
+ }
+
+ private void grantEverySystemPriv(Connector conn, String user) throws AccumuloSecurityException, AccumuloException {
+ SystemPermission[] arrayOfP = new SystemPermission[] {SystemPermission.SYSTEM, SystemPermission.ALTER_TABLE, SystemPermission.ALTER_USER,
+ SystemPermission.CREATE_TABLE, SystemPermission.CREATE_USER, SystemPermission.DROP_TABLE, SystemPermission.DROP_USER};
+ for (SystemPermission p : arrayOfP) {
+ conn.securityOperations().grantSystemPermission(user, p);
+ }
+ }
+
+ @Before
+ public void resetInstance() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException {
+ conn = getConnector();
+
+ // I don't want to recreate the instance for every test since it will take ages.
+ // If we run every test as non-root users, I can drop these users every test which should effectively
+ // reset the environment.
+
+ if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_1))
+ conn.securityOperations().dropLocalUser(AUDIT_USER_1);
+ if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
+ conn.securityOperations().dropLocalUser(AUDIT_USER_2);
+ if (conn.securityOperations().listLocalUsers().contains(AUDIT_USER_2))
+ conn.securityOperations().dropLocalUser(THIRD_TEST_TABLE_NAME);
+ if (conn.tableOperations().exists(NEW_TEST_TABLE_NAME))
+ conn.tableOperations().delete(NEW_TEST_TABLE_NAME);
+ if (conn.tableOperations().exists(OLD_TEST_TABLE_NAME))
+ conn.tableOperations().delete(OLD_TEST_TABLE_NAME);
+
+ // This will set the lastAuditTimestamp for the first test
+ getAuditMessages("setup");
+
+ }
+
+ @Test
+ public void testTableOperationsAudits() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_TABLE);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Testing activity begins here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
+ auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
+ Map<String,String> emptyMap = Collections.emptyMap();
+ Set<String> emptySet = Collections.emptySet();
+ auditConnector.tableOperations().clone(NEW_TEST_TABLE_NAME, OLD_TEST_TABLE_NAME, true, emptyMap, emptySet);
+ auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
+ auditConnector.tableOperations().offline(NEW_TEST_TABLE_NAME);
+ auditConnector.tableOperations().delete(NEW_TEST_TABLE_NAME);
+ // Testing activity ends here
+
+ ArrayList<String> auditMessages = getAuditMessages("testTableOperationsAudits");
+
+ assertEquals(1, findAuditMessage(auditMessages, "action: createTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: renameTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: cloneTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: offlineTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: deleteTable; targetTable: " + NEW_TEST_TABLE_NAME).size());
+
+ }
+
+ @Test
+ public void testUserOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, InterruptedException, IOException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.CREATE_USER);
+ grantEverySystemPriv(conn, AUDIT_USER_1);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Start testing activities here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.securityOperations().createLocalUser(AUDIT_USER_2, new PasswordToken(PASSWORD));
+
+ // It seems only root can grant stuff.
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
+ conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
+ auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
+ conn.securityOperations().grantTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
+ conn.securityOperations().revokeTablePermission(AUDIT_USER_2, NEW_TEST_TABLE_NAME, TablePermission.READ);
+ auditConnector.securityOperations().changeLocalUserPassword(AUDIT_USER_2, new PasswordToken("anything"));
+ auditConnector.securityOperations().changeUserAuthorizations(AUDIT_USER_2, auths);
+ auditConnector.securityOperations().dropLocalUser(AUDIT_USER_2);
+ // Stop testing activities here
+
+ ArrayList<String> auditMessages = getAuditMessages("testUserOperationsAudits");
+
+ assertEquals(1, findAuditMessage(auditMessages, "action: createUser; targetUser: " + AUDIT_USER_2).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: grantSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: revokeSystemPermission; permission: " + SystemPermission.ALTER_TABLE.toString() + "; targetUser: " + AUDIT_USER_2).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: grantTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "action: revokeTablePermission; permission: " + TablePermission.READ.toString() + "; targetTable: " + NEW_TEST_TABLE_NAME).size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: changePassword; targetUser: " + AUDIT_USER_2 + "").size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: changeAuthorizations; targetUser: " + AUDIT_USER_2 + "; authorizations: " + auths.toString())
+ .size());
+ assertEquals(1, findAuditMessage(auditMessages, "action: dropUser; targetUser: " + AUDIT_USER_2).size());
+ }
+
+ @Test
+ public void testImportExportOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException,
+ IOException, InterruptedException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
+ grantEverySystemPriv(conn, AUDIT_USER_1);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Start testing activities here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
+
+ // Insert some play data
+ BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
+ Mutation m = new Mutation("myRow");
+ m.put("cf1", "cq1", "v1");
+ m.put("cf1", "cq2", "v3");
+ bw.addMutation(m);
+ bw.close();
+
+ // Prepare to export the table
+ File exportDir = new File(getCluster().getConfig().getDir().toString() + "/export");
+
+ auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
+ auditConnector.tableOperations().exportTable(OLD_TEST_TABLE_NAME, exportDir.toString());
+
+ // We've exported the table metadata to the MiniAccumuloCluster root dir. Grab the .rf file path to re-import it
+ File distCpTxt = new File(exportDir.toString() + "/distcp.txt");
+ File importFile = null;
- LineIterator it = FileUtils.lineIterator(distCpTxt, Constants.UTF8.name());
++ LineIterator it = FileUtils.lineIterator(distCpTxt, UTF_8.name());
+
+ // Just grab the first rf file, it will do for now.
+ String filePrefix = "file:";
+ try {
+ while (it.hasNext() && importFile == null) {
+ String line = it.nextLine();
+ if (line.matches(".*\\.rf")) {
+ importFile = new File(line.replaceFirst(filePrefix, ""));
+ }
+ }
+ } finally {
+ LineIterator.closeQuietly(it);
+ }
+ FileUtils.copyFileToDirectory(importFile, exportDir);
+ auditConnector.tableOperations().importTable(NEW_TEST_TABLE_NAME, exportDir.toString());
+
+ // Now do a Directory (bulk) import of the same data.
+ auditConnector.tableOperations().create(THIRD_TEST_TABLE_NAME);
+ File failDir = new File(exportDir + "/tmp");
+ failDir.mkdirs();
+ auditConnector.tableOperations().importDirectory(THIRD_TEST_TABLE_NAME, exportDir.toString(), failDir.toString(), false);
+ auditConnector.tableOperations().online(OLD_TEST_TABLE_NAME);
+
+ // Stop testing activities here
+
+ ArrayList<String> auditMessages = getAuditMessages("testImportExportOperationsAudits");
+
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME)).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
+ .size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_EXPORT_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, exportDir.toString())).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ String.format(AuditedSecurityOperation.CAN_IMPORT_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME, filePrefix + exportDir.toString())).size());
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME)).size());
+ assertEquals(
+ 1,
+ findAuditMessage(
+ auditMessages,
+ String.format(AuditedSecurityOperation.CAN_BULK_IMPORT_AUDIT_TEMPLATE, THIRD_TEST_TABLE_NAME, filePrefix + exportDir.toString(), filePrefix
+ + failDir.toString())).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "onlineTable", OLD_TEST_TABLE_NAME))
+ .size());
+
+ }
+
+ @Test
+ public void testDataOperationsAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.securityOperations().grantSystemPermission(AUDIT_USER_1, SystemPermission.SYSTEM);
+ conn.securityOperations().changeUserAuthorizations(AUDIT_USER_1, auths);
+ grantEverySystemPriv(conn, AUDIT_USER_1);
+
+ // Connect as Audit User and do a bunch of stuff.
+ // Start testing activities here
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+ auditConnector.tableOperations().create(OLD_TEST_TABLE_NAME);
+
+ // Insert some play data
+ BatchWriter bw = auditConnector.createBatchWriter(OLD_TEST_TABLE_NAME, new BatchWriterConfig());
+ Mutation m = new Mutation("myRow");
+ m.put("cf1", "cq1", "v1");
+ m.put("cf1", "cq2", "v3");
+ bw.addMutation(m);
+ bw.close();
+
+ // Start testing activities here
+ // A regular scan
+ Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
+ for (Map.Entry<Key,Value> entry : scanner) {
+ System.out.println("Scanner row: " + entry.getKey() + " " + entry.getValue());
+ }
+ scanner.close();
+
+ // A batch scan
+ BatchScanner bs = auditConnector.createBatchScanner(OLD_TEST_TABLE_NAME, auths, 1);
+ bs.fetchColumn(new Text("cf1"), new Text("cq1"));
+ bs.setRanges(Arrays.asList(new Range("myRow", "myRow~")));
+
+ for (Map.Entry<Key,Value> entry : bs) {
+ System.out.println("BatchScanner row: " + entry.getKey() + " " + entry.getValue());
+ }
+ bs.close();
+
+ // Delete some data.
+ auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
+
+ // End of testing activities
+
+ ArrayList<String> auditMessages = getAuditMessages("testDataOperationsAudits");
+ assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertTrue(1 <= findAuditMessage(auditMessages, "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
+
+ }
+
+ @Test
+ public void testDeniedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ // Create our user with no privs
+ conn.securityOperations().createLocalUser(AUDIT_USER_1, new PasswordToken(PASSWORD));
+ conn.tableOperations().create(OLD_TEST_TABLE_NAME);
+ auditConnector = getCluster().getConnector(AUDIT_USER_1, PASSWORD);
+
+ // Start testing activities
+ // We should get denied or / failed audit messages here.
+ // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
+
+ try {
+ auditConnector.tableOperations().create(NEW_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().rename(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().clone(OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME, true, Collections.<String,String> emptyMap(),
+ Collections.<String> emptySet());
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().delete(OLD_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ auditConnector.tableOperations().offline(OLD_TEST_TABLE_NAME);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ Scanner scanner = auditConnector.createScanner(OLD_TEST_TABLE_NAME, auths);
+ scanner.iterator().next().getKey();
+ } catch (RuntimeException ex) {}
+ try {
+ auditConnector.tableOperations().deleteRows(OLD_TEST_TABLE_NAME, new Text("myRow"), new Text("myRow~"));
+ } catch (AccumuloSecurityException ex) {}
+
+ // ... that will do for now.
+ // End of testing activities
+
+ ArrayList<String> auditMessages = getAuditMessages("testDeniedAudits");
+ assertEquals(1,
+ findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CREATE_TABLE_AUDIT_TEMPLATE, NEW_TEST_TABLE_NAME))
+ .size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_RENAME_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_CLONE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, NEW_TEST_TABLE_NAME)).size());
+ assertEquals(1,
+ findAuditMessage(auditMessages, "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_TABLE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME))
+ .size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE, "offlineTable", OLD_TEST_TABLE_NAME))
+ .size());
+ assertEquals(1, findAuditMessage(auditMessages, "operation: denied;.*" + "action: scan; targetTable: " + OLD_TEST_TABLE_NAME).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ "operation: denied;.*" + String.format(AuditedSecurityOperation.CAN_DELETE_RANGE_AUDIT_TEMPLATE, OLD_TEST_TABLE_NAME, "myRow", "myRow~")).size());
+ }
+
+ @Test
+ public void testFailedAudits() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException,
+ InterruptedException {
+
+ // Start testing activities
+ // Test that we get a few "failed" audit messages come through when we tell it to do dumb stuff
+ // We don't want the thrown exceptions to stop our tests, and we are not testing that the Exceptions are thrown.
+ try {
+ conn.securityOperations().dropLocalUser(AUDIT_USER_2);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ conn.securityOperations().revokeSystemPermission(AUDIT_USER_2, SystemPermission.ALTER_TABLE);
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ conn.securityOperations().createLocalUser("root", new PasswordToken("super secret"));
+ } catch (AccumuloSecurityException ex) {}
+ ArrayList<String> auditMessages = getAuditMessages("testFailedAudits");
+ // ... that will do for now.
+ // End of testing activities
+
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.DROP_USER_AUDIT_TEMPLATE, AUDIT_USER_2)).size());
+ assertEquals(
+ 1,
+ findAuditMessage(auditMessages,
+ String.format(AuditedSecurityOperation.REVOKE_SYSTEM_PERMISSION_AUDIT_TEMPLATE, SystemPermission.ALTER_TABLE, AUDIT_USER_2)).size());
+ assertEquals(1, findAuditMessage(auditMessages, String.format(AuditedSecurityOperation.CREATE_USER_AUDIT_TEMPLATE, "root", "")).size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index c1e779e,0000000..8188deb
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@@ -1,1248 -1,0 +1,1248 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.NamespaceExistsException;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.hadoop.io.Text;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class NamespacesIT extends SimpleMacIT {
+
+ private Connector c;
+ private String namespace;
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Before
+ public void setUpConnectorAndNamespace() throws Exception {
+ // prepare a unique namespace and get a new root connector for each test
+ c = getConnector();
+ namespace = "ns_" + getUniqueNames(1)[0];
+ }
+
+ @After
+ public void swingMjölnir() throws Exception {
+ // clean up any added tables, namespaces, and users, after each test
+ for (String t : c.tableOperations().list())
+ if (!Tables.qualify(t).getFirst().equals(Namespaces.ACCUMULO_NAMESPACE))
+ c.tableOperations().delete(t);
+ assertEquals(2, c.tableOperations().list().size());
+ for (String n : c.namespaceOperations().list())
+ if (!n.equals(Namespaces.ACCUMULO_NAMESPACE) && !n.equals(Namespaces.DEFAULT_NAMESPACE))
+ c.namespaceOperations().delete(n);
+ assertEquals(2, c.namespaceOperations().list().size());
+ for (String u : c.securityOperations().listLocalUsers())
+ if (!"root".equals(u))
+ c.securityOperations().dropLocalUser(u);
+ assertEquals(1, c.securityOperations().listLocalUsers().size());
+ }
+
+ @Test
+ public void checkReservedNamespaces() throws Exception {
+ assertEquals(c.namespaceOperations().defaultNamespace(), Namespaces.DEFAULT_NAMESPACE);
+ assertEquals(c.namespaceOperations().systemNamespace(), Namespaces.ACCUMULO_NAMESPACE);
+ }
+
+ @Test
+ public void checkBuiltInNamespaces() throws Exception {
+ assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
+ assertTrue(c.namespaceOperations().exists(Namespaces.ACCUMULO_NAMESPACE));
+ }
+
+ @Test
+ public void createTableInDefaultNamespace() throws Exception {
+ String tableName = "1";
+ c.tableOperations().create(tableName);
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ @Test(expected = AccumuloException.class)
+ public void createTableInAccumuloNamespace() throws Exception {
+ String tableName = Namespaces.ACCUMULO_NAMESPACE + ".1";
+ assertFalse(c.tableOperations().exists(tableName));
+ c.tableOperations().create(tableName); // should fail
+ }
+
+ @Test(expected = AccumuloSecurityException.class)
+ public void deleteDefaultNamespace() throws Exception {
+ c.namespaceOperations().delete(Namespaces.DEFAULT_NAMESPACE); // should fail
+ }
+
+ @Test(expected = AccumuloSecurityException.class)
+ public void deleteAccumuloNamespace() throws Exception {
+ c.namespaceOperations().delete(Namespaces.ACCUMULO_NAMESPACE); // should fail
+ }
+
+ @Test
+ public void createTableInMissingNamespace() throws Exception {
+ String t = namespace + ".1";
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t));
+ try {
+ c.tableOperations().create(t);
+ fail();
+ } catch (AccumuloException e) {
+ assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t));
+ }
+ }
+
+ @Test
+ public void createAndDeleteNamespace() throws Exception {
+ String t1 = namespace + ".1";
+ String t2 = namespace + ".2";
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ try {
+ c.namespaceOperations().delete(namespace);
+ } catch (NamespaceNotFoundException e) {}
+ try {
+ c.tableOperations().delete(t1);
+ } catch (TableNotFoundException e) {
+ assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
+ }
+ c.namespaceOperations().create(namespace);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.tableOperations().create(t1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.tableOperations().create(t2);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t2));
+ c.tableOperations().delete(t1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t2));
+ c.tableOperations().delete(t2);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.namespaceOperations().delete(namespace);
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ }
+
+ @Test(expected = NamespaceNotEmptyException.class)
+ public void deleteNonEmptyNamespace() throws Exception {
+ String tableName1 = namespace + ".1";
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(tableName1));
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(tableName1));
+ c.namespaceOperations().delete(namespace); // should fail
+ }
+
+ @Test
+ public void verifyPropertyInheritance() throws Exception {
+ String t0 = "0";
+ String t1 = namespace + ".1";
+ String t2 = namespace + ".2";
+
+ String k = Property.TABLE_SCAN_MAXMEM.getKey();
+ String v = "42K";
+
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ c.tableOperations().create(t0);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t0));
+
+ // verify no property
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(t1, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // set property and verify
+ c.namespaceOperations().setProperty(namespace, k, v);
+ assertTrue(checkNamespaceHasProp(namespace, k, v));
+ assertTrue(checkTableHasProp(t1, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // add a new table to namespace and verify
+ assertFalse(c.tableOperations().exists(t2));
+ c.tableOperations().create(t2);
+ assertTrue(c.tableOperations().exists(t2));
+ assertTrue(checkNamespaceHasProp(namespace, k, v));
+ assertTrue(checkTableHasProp(t1, k, v));
+ assertTrue(checkTableHasProp(t2, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // remove property and verify
+ c.namespaceOperations().removeProperty(namespace, k);
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(t1, k, v));
+ assertFalse(checkTableHasProp(t2, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(t0, k, v));
+
+ // set property on default namespace and verify
+ c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, k, v);
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(t1, k, v));
+ assertFalse(checkTableHasProp(t2, k, v));
+ assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertTrue(checkTableHasProp(t0, k, v));
+
+ // test that table properties override namespace properties
+ String k2 = Property.TABLE_FILE_MAX.getKey();
+ String v2 = "42";
+ String table_v2 = "13";
+
+ // set new property on some
+ c.namespaceOperations().setProperty(namespace, k2, v2);
+ c.tableOperations().setProperty(t2, k2, table_v2);
+ assertTrue(checkNamespaceHasProp(namespace, k2, v2));
+ assertTrue(checkTableHasProp(t1, k2, v2));
+ assertTrue(checkTableHasProp(t2, k2, table_v2));
+
+ c.tableOperations().delete(t1);
+ c.tableOperations().delete(t2);
+ c.tableOperations().delete(t0);
+ c.namespaceOperations().delete(namespace);
+ }
+
+ @Test
+ public void verifyIteratorInheritance() throws Exception {
+ String t1 = namespace + ".1";
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ String iterName = namespace + "_iter";
+
+ BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes()));
+ bw.addMutation(m);
+ bw.flush();
+ bw.close();
+
+ IteratorSetting setting = new IteratorSetting(250, iterName, SimpleFilter.class.getName());
+
+ // verify can see inserted entry
+ Scanner s = c.createScanner(t1, Authorizations.EMPTY);
+ assertTrue(s.iterator().hasNext());
+ assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
+ assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+
+ // verify entry is filtered out (also, verify conflict checking API)
+ c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
+ c.namespaceOperations().attachIterator(namespace, setting);
+ UtilWaitThread.sleep(2*1000);
+ try {
+ c.namespaceOperations().checkIteratorConflicts(namespace, setting, EnumSet.allOf(IteratorScope.class));
+ fail();
+ } catch (AccumuloException e) {
+ assertEquals(IllegalArgumentException.class.getName(), e.getCause().getClass().getName());
+ }
+ IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan);
+ assertEquals(setting, setting2);
+ assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
+ assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertFalse(s.iterator().hasNext());
+
+ // verify can see inserted entry again
+ c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
+ UtilWaitThread.sleep(2*1000);
+ assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
+ assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertTrue(s.iterator().hasNext());
+ }
+
+ @Test
+ public void cloneTable() throws Exception {
+ String namespace2 = namespace + "_clone";
+ String t1 = namespace + ".1";
+ String t2 = namespace + ".2";
+ String t3 = namespace2 + ".2";
+ String k1 = Property.TABLE_FILE_MAX.getKey();
+ String k2 = Property.TABLE_FILE_REPLICATION.getKey();
+ String k1v1 = "55";
+ String k1v2 = "66";
+ String k2v1 = "5";
+ String k2v2 = "6";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+
+ try {
+ // try to clone before namespace exists
+ c.tableOperations().clone(t1, t3, false, null, null); // should fail
+ fail();
+ } catch (AccumuloException e) {
+ assertEquals(NamespaceNotFoundException.class.getName(), e.getCause().getClass().getName());
+ }
+
+ // try to clone before when target tables exist
+ c.namespaceOperations().create(namespace2);
+ c.tableOperations().create(t2);
+ c.tableOperations().create(t3);
+ for (String t : Arrays.asList(t2, t3)) {
+ try {
+ c.tableOperations().clone(t1, t, false, null, null); // should fail
+ fail();
+ } catch (TableExistsException e) {
+ c.tableOperations().delete(t);
+ }
+ }
+
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+
+ // set property with different values in two namespaces and a separate property with different values on the table and both namespaces
+ assertFalse(checkNamespaceHasProp(namespace, k1, k1v1));
+ assertFalse(checkNamespaceHasProp(namespace2, k1, k1v2));
+ assertFalse(checkTableHasProp(t1, k1, k1v1));
+ assertFalse(checkTableHasProp(t1, k1, k1v2));
+ assertFalse(checkNamespaceHasProp(namespace, k2, k2v1));
+ assertFalse(checkNamespaceHasProp(namespace2, k2, k2v1));
+ assertFalse(checkTableHasProp(t1, k2, k2v1));
+ assertFalse(checkTableHasProp(t1, k2, k2v2));
+ c.namespaceOperations().setProperty(namespace, k1, k1v1);
+ c.namespaceOperations().setProperty(namespace2, k1, k1v2);
+ c.namespaceOperations().setProperty(namespace, k2, k2v1);
+ c.namespaceOperations().setProperty(namespace2, k2, k2v1);
+ c.tableOperations().setProperty(t1, k2, k2v2);
+ assertTrue(checkNamespaceHasProp(namespace, k1, k1v1));
+ assertTrue(checkNamespaceHasProp(namespace2, k1, k1v2));
+ assertTrue(checkTableHasProp(t1, k1, k1v1));
+ assertFalse(checkTableHasProp(t1, k1, k1v2));
+ assertTrue(checkNamespaceHasProp(namespace, k2, k2v1));
+ assertTrue(checkNamespaceHasProp(namespace2, k2, k2v1));
+ assertFalse(checkTableHasProp(t1, k2, k2v1));
+ assertTrue(checkTableHasProp(t1, k2, k2v2));
+
+ // clone twice, once in same namespace, once in another
+ for (String t : Arrays.asList(t2, t3))
+ c.tableOperations().clone(t1, t, false, null, null);
+
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(c.tableOperations().exists(t1));
+ assertTrue(c.tableOperations().exists(t2));
+ assertTrue(c.tableOperations().exists(t3));
+
+ // verify the properties got transferred
+ assertTrue(checkTableHasProp(t1, k1, k1v1));
+ assertTrue(checkTableHasProp(t2, k1, k1v1));
+ assertTrue(checkTableHasProp(t3, k1, k1v2));
+ assertTrue(checkTableHasProp(t1, k2, k2v2));
+ assertTrue(checkTableHasProp(t2, k2, k2v2));
+ assertTrue(checkTableHasProp(t3, k2, k2v2));
+ }
+
+ @Test
+ public void renameNamespaceWithTable() throws Exception {
+ String namespace2 = namespace + "_renamed";
+ String t1 = namespace + ".t";
+ String t2 = namespace2 + ".t";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1);
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t2));
+
+ String namespaceId = c.namespaceOperations().namespaceIdMap().get(namespace);
+ String tableId = c.tableOperations().tableIdMap().get(t1);
+
+ c.namespaceOperations().rename(namespace, namespace2);
+ assertFalse(c.namespaceOperations().exists(namespace));
+ assertFalse(c.tableOperations().exists(t1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(c.tableOperations().exists(t2));
+
+ // verify id's didn't change
+ String namespaceId2 = c.namespaceOperations().namespaceIdMap().get(namespace2);
+ String tableId2 = c.tableOperations().tableIdMap().get(t2);
+
+ assertEquals(namespaceId, namespaceId2);
+ assertEquals(tableId, tableId2);
+ }
+
+ @Test
+ public void verifyConstraintInheritance() throws Exception {
+ String t1 = namespace + ".1";
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t1, false);
+ String constraintClassName = NumericValueConstraint.class.getName();
+
+ assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
+ assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
+
+ c.namespaceOperations().addConstraint(namespace, constraintClassName);
+ assertTrue(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
+ assertTrue(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
+ int num = c.namespaceOperations().listConstraints(namespace).get(constraintClassName);
+ assertEquals(num, (int) c.tableOperations().listConstraints(t1).get(constraintClassName));
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+
+ Mutation m1 = new Mutation("r1");
+ Mutation m2 = new Mutation("r2");
+ Mutation m3 = new Mutation("r3");
- m1.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
- m2.put("e", "f", new Value("123".getBytes(Constants.UTF8)));
- m3.put("c", "d", new Value("zyxwv".getBytes(Constants.UTF8)));
++ m1.put("a", "b", new Value("abcde".getBytes(UTF_8)));
++ m2.put("e", "f", new Value("123".getBytes(UTF_8)));
++ m3.put("c", "d", new Value("zyxwv".getBytes(UTF_8)));
+ BatchWriter bw = c.createBatchWriter(t1, new BatchWriterConfig());
+ bw.addMutations(Arrays.asList(m1, m2, m3));
+ try {
+ bw.close();
+ fail();
+ } catch (MutationsRejectedException e) {
+ assertEquals(1, e.getConstraintViolationSummaries().size());
+ assertEquals(2, e.getConstraintViolationSummaries().get(0).getNumberOfViolatingMutations());
+ }
+ c.namespaceOperations().removeConstraint(namespace, num);
+ assertFalse(c.namespaceOperations().listConstraints(namespace).containsKey(constraintClassName));
+ assertFalse(c.tableOperations().listConstraints(t1).containsKey(constraintClassName));
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+
+ bw = c.createBatchWriter(t1, new BatchWriterConfig());
+ bw.addMutations(Arrays.asList(m1, m2, m3));
+ bw.close();
+ }
+
+ @Test
+ public void renameTable() throws Exception {
+ String namespace2 = namespace + "_renamed";
+ String t1 = namespace + ".1";
+ String t2 = namespace2 + ".2";
+ String t3 = namespace + ".3";
+ String t4 = namespace + ".4";
+ String t5 = "5";
+
+ c.namespaceOperations().create(namespace);
+ c.namespaceOperations().create(namespace2);
+
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+ assertFalse(c.tableOperations().exists(t4));
+ assertFalse(c.tableOperations().exists(t5));
+
+ c.tableOperations().create(t1);
+
+ try {
+ c.tableOperations().rename(t1, t2);
+ fail();
+ } catch (AccumuloException e) {
+ // this is expected, because we don't allow renames across namespaces
+ assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
+ assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
+ assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
+ }
+
+ try {
+ c.tableOperations().rename(t1, t5);
+ fail();
+ } catch (AccumuloException e) {
+ // this is expected, because we don't allow renames across namespaces
+ assertEquals(ThriftTableOperationException.class.getName(), e.getCause().getClass().getName());
+ assertEquals(TableOperation.RENAME, ((ThriftTableOperationException) e.getCause()).getOp());
+ assertEquals(TableOperationExceptionType.INVALID_NAME, ((ThriftTableOperationException) e.getCause()).getType());
+ }
+
+ assertTrue(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ assertFalse(c.tableOperations().exists(t3));
+ assertFalse(c.tableOperations().exists(t4));
+ assertFalse(c.tableOperations().exists(t5));
+
+ // fully qualified rename
+ c.tableOperations().rename(t1, t3);
+ assertFalse(c.tableOperations().exists(t1));
+ assertFalse(c.tableOperations().exists(t2));
+ assertTrue(c.tableOperations().exists(t3));
+ assertFalse(c.tableOperations().exists(t4));
+ assertFalse(c.tableOperations().exists(t5));
+ }
+
+ /**
+ * Tests new Namespace permissions as well as modifications to Table permissions because of namespaces. Checks each permission to first make sure the user
+ * doesn't have permission to perform the action, then root grants them the permission and we check to make sure they could perform the action.
+ */
+ @Test
+ public void testPermissions() throws Exception {
+ String u1 = "u1";
+ String u2 = "u2";
+ PasswordToken pass = new PasswordToken("pass");
+
+ String n1 = namespace;
+ String t1 = n1 + ".1";
+ String t2 = n1 + ".2";
+ String t3 = n1 + ".3";
+
+ String n2 = namespace + "_2";
+
+ c.namespaceOperations().create(n1);
+ c.tableOperations().create(t1);
+
+ c.securityOperations().createLocalUser(u1, pass);
+
+ Connector user1Con = c.getInstance().getConnector(u1, pass);
+
+ try {
+ user1Con.tableOperations().create(t2);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
+ user1Con.tableOperations().create(t2);
+ assertTrue(c.tableOperations().list().contains(t2));
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.CREATE_TABLE);
+
+ try {
+ user1Con.tableOperations().delete(t1);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
+ user1Con.tableOperations().delete(t1);
+ assertTrue(!c.tableOperations().list().contains(t1));
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.DROP_TABLE);
+
+ c.tableOperations().create(t3);
+ BatchWriter bw = c.createBatchWriter(t3, null);
+ Mutation m = new Mutation("row");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ Iterator<Entry<Key,Value>> i = user1Con.createScanner(t3, new Authorizations()).iterator();
+ try {
+ i.next();
+ fail();
+ } catch (RuntimeException e) {
+ assertEquals(AccumuloSecurityException.class.getName(), e.getCause().getClass().getName());
+ expectPermissionDenied((AccumuloSecurityException) e.getCause());
+ }
+
+ m = new Mutation(u1);
+ m.put("cf", "cq", "turtles");
+ bw = user1Con.createBatchWriter(t3, null);
+ try {
+ bw.addMutation(m);
+ bw.close();
+ fail();
+ } catch (MutationsRejectedException e) {
+ assertEquals(1, e.getAuthorizationFailuresMap().size());
+ assertEquals(1, e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().size());
+ switch (e.getAuthorizationFailuresMap().entrySet().iterator().next().getValue().iterator().next()) {
+ case PERMISSION_DENIED:
+ break;
+ default:
+ fail();
+ }
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.READ);
+ i = user1Con.createScanner(t3, new Authorizations()).iterator();
+ assertTrue(i.hasNext());
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.READ);
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.WRITE);
+ m = new Mutation(u1);
+ m.put("cf", "cq", "turtles");
+ bw = user1Con.createBatchWriter(t3, null);
+ bw.addMutation(m);
+ bw.close();
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.WRITE);
+
+ try {
+ user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
+ user1Con.tableOperations().setProperty(t3, Property.TABLE_FILE_MAX.getKey(), "42");
+ user1Con.tableOperations().removeProperty(t3, Property.TABLE_FILE_MAX.getKey());
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_TABLE);
+
+ try {
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "55");
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "42");
+ user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.ALTER_NAMESPACE);
+
+ c.securityOperations().createLocalUser(u2, pass);
+ try {
+ user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantNamespacePermission(u1, n1, NamespacePermission.GRANT);
+ user1Con.securityOperations().grantNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
+ user1Con.securityOperations().revokeNamespacePermission(u2, n1, NamespacePermission.ALTER_NAMESPACE);
+ c.securityOperations().revokeNamespacePermission(u1, n1, NamespacePermission.GRANT);
+
+ try {
+ user1Con.namespaceOperations().create(n2);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
+ user1Con.namespaceOperations().create(n2);
+ c.securityOperations().revokeSystemPermission(u1, SystemPermission.CREATE_NAMESPACE);
+
+ c.securityOperations().revokeNamespacePermission(u1, n2, NamespacePermission.DROP_NAMESPACE);
+ try {
+ user1Con.namespaceOperations().delete(n2);
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
+ user1Con.namespaceOperations().delete(n2);
+ c.securityOperations().revokeSystemPermission(u1, SystemPermission.DROP_NAMESPACE);
+
+ try {
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
+ fail();
+ } catch (AccumuloSecurityException e) {
+ expectPermissionDenied(e);
+ }
+
+ c.securityOperations().grantSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
+ user1Con.namespaceOperations().setProperty(n1, Property.TABLE_FILE_MAX.getKey(), "33");
+ user1Con.namespaceOperations().removeProperty(n1, Property.TABLE_FILE_MAX.getKey());
+ c.securityOperations().revokeSystemPermission(u1, SystemPermission.ALTER_NAMESPACE);
+ }
+
+ @Test
+ public void verifySystemPropertyInheritance() throws Exception {
+ String t1 = "1";
+ String t2 = namespace + "." + t1;
+ c.tableOperations().create(t1);
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(t2);
+
+ // verify iterator inheritance
+ _verifySystemPropertyInheritance(t1, t2, Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.sum", "20," + SimpleFilter.class.getName(), false);
+
+ // verify constraint inheritance
+ _verifySystemPropertyInheritance(t1, t2, Property.TABLE_CONSTRAINT_PREFIX.getKey() + "42", NumericValueConstraint.class.getName(), false);
+
+ // verify other inheritance
+ _verifySystemPropertyInheritance(t1, t2, Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "dummy", "dummy", true);
+ }
+
+ private void _verifySystemPropertyInheritance(String defaultNamespaceTable, String namespaceTable, String k, String v, boolean systemNamespaceShouldInherit)
+ throws Exception {
+ // nobody should have any of these properties yet
+ assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
+ assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(RootTable.NAME, k, v));
+ assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(namespaceTable, k, v));
+
+ // set the filter, verify that accumulo namespace is the only one unaffected
+ c.instanceOperations().setProperty(k, v);
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+ assertTrue(c.instanceOperations().getSystemConfiguration().containsValue(v));
+ assertEquals(systemNamespaceShouldInherit, checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
+ assertEquals(systemNamespaceShouldInherit, checkTableHasProp(RootTable.NAME, k, v));
+ assertEquals(systemNamespaceShouldInherit, checkTableHasProp(MetadataTable.NAME, k, v));
+ assertTrue(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertTrue(checkTableHasProp(defaultNamespaceTable, k, v));
+ assertTrue(checkNamespaceHasProp(namespace, k, v));
+ assertTrue(checkTableHasProp(namespaceTable, k, v));
+
+ // verify it is no longer inherited
+ c.instanceOperations().removeProperty(k);
+ // doesn't take effect immediately, needs time to propagate to tserver's ZooKeeper cache
+ UtilWaitThread.sleep(250);
+ assertFalse(c.instanceOperations().getSystemConfiguration().containsValue(v));
+ assertFalse(checkNamespaceHasProp(Namespaces.ACCUMULO_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(RootTable.NAME, k, v));
+ assertFalse(checkTableHasProp(MetadataTable.NAME, k, v));
+ assertFalse(checkNamespaceHasProp(Namespaces.DEFAULT_NAMESPACE, k, v));
+ assertFalse(checkTableHasProp(defaultNamespaceTable, k, v));
+ assertFalse(checkNamespaceHasProp(namespace, k, v));
+ assertFalse(checkTableHasProp(namespaceTable, k, v));
+ }
+
+ @Test
+ public void listNamespaces() throws Exception {
+ SortedSet<String> namespaces = c.namespaceOperations().list();
+ Map<String,String> map = c.namespaceOperations().namespaceIdMap();
+ assertEquals(2, namespaces.size());
+ assertEquals(2, map.size());
+ assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
+ assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
+ assertFalse(namespaces.contains(namespace));
+ assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
+ assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
+ assertNull(map.get(namespace));
+
+ c.namespaceOperations().create(namespace);
+ namespaces = c.namespaceOperations().list();
+ map = c.namespaceOperations().namespaceIdMap();
+ assertEquals(3, namespaces.size());
+ assertEquals(3, map.size());
+ assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
+ assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
+ assertTrue(namespaces.contains(namespace));
+ assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
+ assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
+ assertNotNull(map.get(namespace));
+
+ c.namespaceOperations().delete(namespace);
+ namespaces = c.namespaceOperations().list();
+ map = c.namespaceOperations().namespaceIdMap();
+ assertEquals(2, namespaces.size());
+ assertEquals(2, map.size());
+ assertTrue(namespaces.contains(Namespaces.ACCUMULO_NAMESPACE));
+ assertTrue(namespaces.contains(Namespaces.DEFAULT_NAMESPACE));
+ assertFalse(namespaces.contains(namespace));
+ assertEquals(Namespaces.ACCUMULO_NAMESPACE_ID, map.get(Namespaces.ACCUMULO_NAMESPACE));
+ assertEquals(Namespaces.DEFAULT_NAMESPACE_ID, map.get(Namespaces.DEFAULT_NAMESPACE));
+ assertNull(map.get(namespace));
+ }
+
+ @Test
+ public void loadClass() throws Exception {
+ assertTrue(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
+ assertFalse(c.namespaceOperations().testClassLoad(Namespaces.DEFAULT_NAMESPACE, "dummy", SortedKeyValueIterator.class.getName()));
+ try {
+ c.namespaceOperations().testClassLoad(namespace, "dummy", "dummy");
+ fail();
+ } catch (NamespaceNotFoundException e) {}
+ }
+
+ @Test
+ public void testModifyingPermissions() throws Exception {
+ String tableName = namespace + ".modify";
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName);
+ assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
+ c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
+ assertFalse(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
+ c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
+ assertTrue(c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ));
+ c.tableOperations().delete(tableName);
+
+ try {
+ c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
+ c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ assertFalse(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
+ c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ assertTrue(c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ));
+
+ c.namespaceOperations().delete(namespace);
+
+ try {
+ c.securityOperations().hasTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().grantTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().revokeTablePermission(c.whoami(), tableName, TablePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().hasNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().grantNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ try {
+ c.securityOperations().revokeNamespacePermission(c.whoami(), namespace, NamespacePermission.READ);
+ fail();
+ } catch (Exception e) {
+ if (!(e instanceof AccumuloSecurityException) || !((AccumuloSecurityException) e).getSecurityErrorCode().equals(SecurityErrorCode.NAMESPACE_DOESNT_EXIST))
+ throw new Exception("Has permission resulted in " + e.getClass().getName(), e);
+ }
+
+ }
+
+ @Test
+ public void verifyTableOperationsExceptions() throws Exception {
+ String tableName = namespace + ".1";
+ IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
+ Text a = new Text("a");
+ Text z = new Text("z");
+ TableOperations ops = c.tableOperations();
+
+ // this one doesn't throw an exception, so don't fail; just check that it works
+ assertFalse(ops.exists(tableName));
+
+ // table operations that should throw an AccumuloException caused by NamespaceNotFoundException
+ int numRun = 0;
+ ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.create(tableName);
+ fail();
+ case 1:
+ ops.create("a");
+ ops.clone("a", tableName, true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
+ fail();
+ case 2:
+ ops.importTable(tableName, System.getProperty("user.dir") + "/target");
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(3, i); // check test integrity
+ assertEquals(3, numRun); // check test integrity
+ break ACCUMULOEXCEPTIONS_NAMESPACENOTFOUND;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof AccumuloException) || !(e.getCause() instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+
+ // table operations that should throw an AccumuloException caused by a TableNotFoundException caused by a NamespaceNotFoundException
+ // these are here because we didn't declare TableNotFoundException in the API :(
+ numRun = 0;
+ ACCUMULOEXCEPTIONS_TABLENOTFOUND: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.removeConstraint(tableName, 0);
+ fail();
+ case 1:
+ ops.removeProperty(tableName, "a");
+ fail();
+ case 2:
+ ops.setProperty(tableName, "a", "b");
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(3, i); // check test integrity
+ assertEquals(3, numRun); // check test integrity
+ break ACCUMULOEXCEPTIONS_TABLENOTFOUND;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof AccumuloException) || !(e.getCause() instanceof TableNotFoundException)
+ || !(e.getCause().getCause() instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+
+ // table operations that should throw a TableNotFoundException caused by NamespaceNotFoundException
+ numRun = 0;
+ TABLENOTFOUNDEXCEPTIONS: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.addConstraint(tableName, NumericValueConstraint.class.getName());
+ fail();
+ case 1:
+ ops.addSplits(tableName, new TreeSet<Text>());
+ fail();
+ case 2:
+ ops.attachIterator(tableName, setting);
+ fail();
+ case 3:
+ ops.cancelCompaction(tableName);
+ fail();
+ case 4:
+ ops.checkIteratorConflicts(tableName, setting, EnumSet.allOf(IteratorScope.class));
+ fail();
+ case 5:
+ ops.clearLocatorCache(tableName);
+ fail();
+ case 6:
+ ops.clone(tableName, "2", true, Collections.<String,String> emptyMap(), Collections.<String> emptySet());
+ fail();
+ case 7:
+ ops.compact(tableName, a, z, true, true);
+ fail();
+ case 8:
+ ops.delete(tableName);
+ fail();
+ case 9:
+ ops.deleteRows(tableName, a, z);
+ fail();
+ case 10:
+ ops.splitRangeByTablets(tableName, new Range(), 10);
+ fail();
+ case 11:
+ ops.exportTable(tableName, namespace + "_dir");
+ fail();
+ case 12:
+ ops.flush(tableName, a, z, true);
+ fail();
+ case 13:
+ ops.getDiskUsage(Collections.singleton(tableName));
+ fail();
+ case 14:
+ ops.getIteratorSetting(tableName, "a", IteratorScope.scan);
+ fail();
+ case 15:
+ ops.getLocalityGroups(tableName);
+ fail();
+ case 16:
+ ops.getMaxRow(tableName, Authorizations.EMPTY, a, true, z, true);
+ fail();
+ case 17:
+ ops.getProperties(tableName);
+ fail();
+ case 18:
+ ops.importDirectory(tableName, "", "", false);
+ fail();
+ case 19:
+ ops.testClassLoad(tableName, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail();
+ case 20:
+ ops.listConstraints(tableName);
+ fail();
+ case 21:
+ ops.listIterators(tableName);
+ fail();
+ case 22:
+ ops.listSplits(tableName);
+ fail();
+ case 23:
+ ops.merge(tableName, a, z);
+ fail();
+ case 24:
+ ops.offline(tableName, true);
+ fail();
+ case 25:
+ ops.online(tableName, true);
+ fail();
+ case 26:
+ ops.removeIterator(tableName, "a", EnumSet.of(IteratorScope.scan));
+ fail();
+ case 27:
+ ops.rename(tableName, tableName + "2");
+ fail();
+ case 28:
+ ops.setLocalityGroups(tableName, Collections.<String,Set<Text>> emptyMap());
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(29, i); // check test integrity
+ assertEquals(29, numRun); // check test integrity
+ break TABLENOTFOUNDEXCEPTIONS;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof TableNotFoundException) || !(e.getCause() instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+ }
+
+ @Test
+ public void verifyNamespaceOperationsExceptions() throws Exception {
+ IteratorSetting setting = new IteratorSetting(200, VersioningIterator.class);
+ NamespaceOperations ops = c.namespaceOperations();
+
+ // this one doesn't throw an exception, so don't fail; just check that it works
+ assertFalse(ops.exists(namespace));
+
+ // namespace operations that should throw a NamespaceNotFoundException
+ int numRun = 0;
+ NAMESPACENOTFOUND: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.addConstraint(namespace, NumericValueConstraint.class.getName());
+ fail();
+ case 1:
+ ops.attachIterator(namespace, setting);
+ fail();
+ case 2:
+ ops.checkIteratorConflicts(namespace, setting, EnumSet.of(IteratorScope.scan));
+ fail();
+ case 3:
+ ops.delete(namespace);
+ fail();
+ case 4:
+ ops.getIteratorSetting(namespace, "thing", IteratorScope.scan);
+ fail();
+ case 5:
+ ops.getProperties(namespace);
+ fail();
+ case 6:
+ ops.listConstraints(namespace);
+ fail();
+ case 7:
+ ops.listIterators(namespace);
+ fail();
+ case 8:
+ ops.removeConstraint(namespace, 1);
+ fail();
+ case 9:
+ ops.removeIterator(namespace, "thing", EnumSet.allOf(IteratorScope.class));
+ fail();
+ case 10:
+ ops.removeProperty(namespace, "a");
+ fail();
+ case 11:
+ ops.rename(namespace, namespace + "2");
+ fail();
+ case 12:
+ ops.setProperty(namespace, "k", "v");
+ fail();
+ case 13:
+ ops.testClassLoad(namespace, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(14, i); // check test integrity
+ assertEquals(14, numRun); // check test integrity
+ break NAMESPACENOTFOUND;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof NamespaceNotFoundException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+
+ // namespace operations that should throw a NamespaceExistsException
+ numRun = 0;
+ NAMESPACEEXISTS: for (int i = 0;; ++i)
+ try {
+ switch (i) {
+ case 0:
+ ops.create(namespace + "0");
+ ops.create(namespace + "0"); // should fail here
+ fail();
+ case 1:
+ ops.create(namespace + i + "_1");
+ ops.create(namespace + i + "_2");
+ ops.rename(namespace + i + "_1", namespace + i + "_2"); // should fail here
+ fail();
+ case 2:
+ ops.create(Namespaces.DEFAULT_NAMESPACE);
+ fail();
+ case 3:
+ ops.create(Namespaces.ACCUMULO_NAMESPACE);
+ fail();
+ case 4:
+ ops.create(namespace + i + "_1");
+ ops.rename(namespace + i + "_1", Namespaces.DEFAULT_NAMESPACE); // should fail here
+ fail();
+ case 5:
+ ops.create(namespace + i + "_1");
+ ops.rename(namespace + i + "_1", Namespaces.ACCUMULO_NAMESPACE); // should fail here
+ fail();
+ default:
+ // break out of infinite loop
+ assertEquals(6, i); // check test integrity
+ assertEquals(6, numRun); // check test integrity
+ break NAMESPACEEXISTS;
+ }
+ } catch (Exception e) {
+ numRun++;
+ if (!(e instanceof NamespaceExistsException))
+ throw new Exception("Case " + i + " resulted in " + e.getClass().getName(), e);
+ }
+ }
+
+ private boolean checkTableHasProp(String t, String propKey, String propVal) {
+ return checkHasProperty(t, propKey, propVal, true);
+ }
+
+ private boolean checkNamespaceHasProp(String n, String propKey, String propVal) {
+ return checkHasProperty(n, propKey, propVal, false);
+ }
+
+ private boolean checkHasProperty(String name, String propKey, String propVal, boolean nameIsTable) {
+ try {
+ Iterable<Entry<String,String>> iterable = nameIsTable ? c.tableOperations().getProperties(name) : c.namespaceOperations().getProperties(name);
+ for (Entry<String,String> e : iterable)
+ if (propKey.equals(e.getKey()))
+ return propVal.equals(e.getValue());
+ return false;
+ } catch (Exception e) {
+ fail();
+ return false;
+ }
+ }
+
+ public static class SimpleFilter extends Filter {
+ @Override
+ public boolean accept(Key k, Value v) {
+ if (k.getColumnFamily().toString().equals("a"))
+ return false;
+ return true;
+ }
+ }
+
+ private void expectPermissionDenied(AccumuloSecurityException sec) {
+ assertEquals(sec.getSecurityErrorCode().getClass(), SecurityErrorCode.class);
+ switch (sec.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ break;
+ default:
+ fail();
+ }
+ }
+
+}
[40/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index cf1f0ea,0000000..da83386
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,407 -1,0 +1,409 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.6.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN,
+ }
+
+ public static enum TokenSource {
+ FILE, INLINE;
+
+ private String prefix;
+
+ private TokenSource() {
+ prefix = name().toLowerCase() + ":";
+ }
+
+ public String prefix() {
+ return prefix;
+ }
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.6.0
+ */
+ public static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.6.0
+ */
+ public static enum GeneralOpts {
+ LOG_LEVEL,
+ VISIBILITY_CACHE_SIZE
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.6.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, token);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
+ TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
+ * Configuration. Token file created with "bin/accumulo create-token".
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param tokenFile
+ * the path to the token file in DFS
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ ArgumentChecker.notNull(principal, tokenFile);
+
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
+ }
+
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ * @see #setConnectorInfo(Class, Configuration, String, String)
+ */
+ public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
+ String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
+ if (token == null || token.isEmpty())
+ return null;
+ if (token.startsWith(TokenSource.INLINE.prefix())) {
+ String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
+ if (args.length == 2)
- return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(Constants.UTF8)));
++ return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(UTF_8)));
+ } else if (token.startsWith(TokenSource.FILE.prefix())) {
+ String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
+ return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
+ }
+
+ throw new IllegalStateException("Token was not properly serialized into the configuration");
+ }
+
+ /**
+ * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
+ *
+ * @param conf
+ * the Hadoop context for the configured job
+ * @return path to the token file as a String
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
+ FSDataInputStream in = null;
+ try {
+ URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
+ Path path = null;
+ for (URI u : uris) {
+ if (u.toString().equals(tokenFile)) {
+ path = new Path(u);
+ }
+ }
+ if (path == null) {
+ throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
+ }
+ FileSystem fs = FileSystem.get(conf);
+ in = fs.open(path);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
+ }
+ java.util.Scanner fileScanner = new java.util.Scanner(in);
+ try {
+ while (fileScanner.hasNextLine()) {
+ Credentials creds = Credentials.deserialize(fileScanner.nextLine());
+ if (principal.equals(creds.getPrincipal())) {
+ return creds.getToken();
+ }
+ }
+ throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
+ } finally {
+ if (fileScanner != null && fileScanner.ioException() == null)
+ fileScanner.close();
+ else if (fileScanner.ioException() != null)
+ throw new RuntimeException(fileScanner.ioException());
+ }
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+ if (clientConfig != null) {
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
+ }
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.6.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ ArgumentChecker.notNull(instanceName);
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.6.0
+ * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+ if (clientConfigString == null) {
+ String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+ String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
+ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
+ } else {
+ return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
+ }
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.6.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ ArgumentChecker.notNull(level);
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.6.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
index 1ed23e3,0000000..f8a5d03
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@@ -1,795 -1,0 +1,795 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.core.util.ArgumentChecker.notNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
+import org.apache.accumulo.core.client.mock.impl.MockTabletLocator;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @since 1.6.0
+ */
+public class InputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Scanner}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ScanOpts {
+ TABLE_NAME, AUTHORIZATIONS, RANGES, COLUMNS, ITERATORS, TABLE_CONFIGS
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ AUTO_ADJUST_RANGES, SCAN_ISOLATION, USE_LOCAL_ITERATORS, SCAN_OFFLINE
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
+ notNull(tableName);
+ conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @since 1.6.0
+ */
+ public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param auths
+ * the user's authorizations
+ * @since 1.6.0
+ */
+ public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf, Authorizations auths) {
+ if (auths != null && !auths.isEmpty())
+ conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the Accumulo scan authorizations
+ * @since 1.6.0
+ * @see #setScanAuthorizations(Class, Configuration, Authorizations)
+ */
+ public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
+ String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
- return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(Constants.UTF8));
++ return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(UTF_8));
+ }
+
+ /**
+ * Sets the input ranges to scan on all input tables for this job. If not set, the entire table will be scanned.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param ranges
+ * the ranges that will be mapped over
+ * @throws IllegalArgumentException
+ * if the ranges cannot be encoded into base 64
+ * @since 1.6.0
+ */
+ public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
+ notNull(ranges);
+
+ ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
+ try {
+ for (Range r : ranges) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ r.write(new DataOutputStream(baos));
+ rangeStrings.add(Base64.encodeBase64String(baos.toByteArray()));
+ }
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES), rangeStrings.toArray(new String[0]));
+ } catch (IOException ex) {
+ throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
+ }
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.6.0
+ * @see #setRanges(Class, Configuration, Collection)
+ */
+ public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
+
+ Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
+ List<Range> ranges = new ArrayList<Range>();
+ for (String rangeString : encodedRanges) {
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes()));
+ Range range = new Range();
+ range.readFields(new DataInputStream(bais));
+ ranges.add(range);
+ }
+ return ranges;
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a list of iterators
+ * @since 1.6.0
+ * @see #addIterator(Class, Configuration, IteratorSetting)
+ */
+ public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
+ String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
+
+ // If no iterators are present, return an empty list
+ if (iterators == null || iterators.isEmpty())
+ return new ArrayList<IteratorSetting>();
+
+ // Compose the set of iterators encoded in the job configuration
+ StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
+ List<IteratorSetting> list = new ArrayList<IteratorSetting>();
+ try {
+ while (tokens.hasMoreTokens()) {
+ String itstring = tokens.nextToken();
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
+ list.add(new IteratorSetting(new DataInputStream(bais)));
+ bais.close();
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("couldn't decode iterator settings");
+ }
+ return list;
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for the single input table on this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @throws IllegalArgumentException
+ * if the column family is null
+ * @since 1.6.0
+ */
+ public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ notNull(columnFamilyColumnQualifierPairs);
+ String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
+ }
+
+ public static String[] serializeColumns(Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ notNull(columnFamilyColumnQualifierPairs);
+ ArrayList<String> columnStrings = new ArrayList<String>(columnFamilyColumnQualifierPairs.size());
+ for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
+
+ if (column.getFirst() == null)
+ throw new IllegalArgumentException("Column family can not be null");
+
+ String col = Base64.encodeBase64String(TextUtil.getBytes(column.getFirst()));
+ if (column.getSecond() != null)
+ col += ":" + Base64.encodeBase64String(TextUtil.getBytes(column.getSecond()));
+ columnStrings.add(col);
+ }
+
+ return columnStrings.toArray(new String[0]);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a set of columns
+ * @since 1.6.0
+ * @see #fetchColumns(Class, Configuration, Collection)
+ */
+ public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass, Configuration conf) {
+ notNull(conf);
+ String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
+ List<String> serialized = new ArrayList<String>();
+ if (confValue != null) {
+ // Split and include any trailing empty strings to allow empty column families
+ for (String val : confValue.split(",", -1)) {
+ serialized.add(val);
+ }
+ }
+ return deserializeFetchedColumns(serialized);
+ }
+
+ public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
+ Set<Pair<Text,Text>> columns = new HashSet<Pair<Text,Text>>();
+
+ if (null == serialized) {
+ return columns;
+ }
+
+ for (String col : serialized) {
+ int idx = col.indexOf(":");
- Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8)));
- Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8)));
++ Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(UTF_8)));
++ Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(UTF_8)));
+ columns.add(new Pair<Text,Text>(cf, cq));
+ }
+ return columns;
+ }
+
+ /**
+ * Encode an iterator on the input for the single input table associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param cfg
+ * the configuration of the iterator
+ * @throws IllegalArgumentException
+ * if the iterator can't be serialized into the configuration
+ * @since 1.6.0
+ */
+ public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String newIter;
+ try {
+ cfg.write(new DataOutputStream(baos));
+ newIter = Base64.encodeBase64String(baos.toByteArray());
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize IteratorSetting");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
+ String iterators = conf.get(confKey);
+ // No iterators specified yet, create a new string
+ if (iterators == null || iterators.isEmpty()) {
+ iterators = newIter;
+ } else {
+ // append the next iterator & reset
+ iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
+ }
+ // Store the iterators w/ the job
+ conf.set(confKey, iterators);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Class, Configuration, Collection)
+ * @since 1.6.0
+ */
+ public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(Class, Configuration, boolean)
+ */
+ public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setScanIsolation(Class, Configuration, boolean)
+ */
+ public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setLocalIterators(Class, Configuration, boolean)
+ */
+ public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineTableScan(Class, Configuration, boolean)
+ */
+ public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
+ }
+
+ /**
+ * Sets configurations for multiple tables at a time.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param configs
+ * an array of {@link InputTableConfig} objects to associate with the job
+ * @since 1.6.0
+ */
+ public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String,InputTableConfig> configs) {
+ MapWritable mapWritable = new MapWritable();
+ for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
+ mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ mapWritable.write(new DataOutputStream(baos));
+ } catch (IOException e) {
+ throw new IllegalStateException("Table configuration could not be serialized.");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
+ conf.set(confKey, Base64.encodeBase64String(baos.toByteArray()));
+ }
+
+ /**
+ * Returns all {@link InputTableConfig} objects associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return all of the table query configs for the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
+ Map<String,InputTableConfig> configs = new HashMap<String,InputTableConfig>();
+ Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
+ if (defaultConfig != null)
+ configs.put(defaultConfig.getKey(), defaultConfig.getValue());
+ String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
+ MapWritable mapWritable = new MapWritable();
+ if (configString != null) {
+ try {
+ byte[] bytes = Base64.decodeBase64(configString.getBytes());
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ mapWritable.readFields(new DataInputStream(bais));
+ bais.close();
+ } catch (IOException e) {
+ throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
+ }
+ }
+ for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
+ configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());
+
+ return configs;
+ }
+
+ /**
+ * Returns the {@link InputTableConfig} for the given table
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table name for which to fetch the table query config
+ * @return the table query config for the given table name (if it exists) and null if it does not
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf, String tableName) {
+ Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
+ return queryConfigs.get(tableName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableId
+ * The table id for which to initialize the {@link TabletLocator}
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf, String tableId) throws TableNotFoundException {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
+ if ("MockInstance".equals(instanceType))
+ return new MockTabletLocator();
+ Instance instance = getInstance(implementingClass, conf);
+ return TabletLocator.getLocator(instance, new Text(tableId));
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.6.0
+ */
+ public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
+
+ Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
+ if (!isConnectorInfoSet(implementingClass, conf))
+ throw new IOException("Input info has not been set.");
+ String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
+ if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
+ throw new IOException("Instance info has not been set.");
+ // validate that we can connect as configured
+ try {
+ String principal = getPrincipal(implementingClass, conf);
+ AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
+ Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
+ if (!c.securityOperations().authenticateUser(principal, token))
+ throw new IOException("Unable to authenticate user");
+
+ if (getInputTableConfigs(implementingClass, conf).size() == 0)
+ throw new IOException("No table set.");
+
+ for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
+ if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
+ throw new IOException("Unable to access table");
+ }
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+ if (!tableConfig.shouldUseLocalIterators()) {
+ if (tableConfig.getIterators() != null) {
+ for (IteratorSetting iter : tableConfig.getIterators()) {
+ if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
+ throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
+ }
+ }
+ }
+ }
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the configuration based on the properties set using the single-table
+ * input methods.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop instance for which to retrieve the configuration
+ * @return the config object built from the single input table properties set on the job
+ * @since 1.6.0
+ */
+ protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(Class<?> implementingClass, Configuration conf) {
+ String tableName = getInputTableName(implementingClass, conf);
+ if (tableName != null) {
+ InputTableConfig queryConfig = new InputTableConfig();
+ List<IteratorSetting> itrs = getIterators(implementingClass, conf);
+ if (itrs != null)
+ queryConfig.setIterators(itrs);
+ Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
+ if (columns != null)
+ queryConfig.fetchColumns(columns);
+ List<Range> ranges = null;
+ try {
+ ranges = getRanges(implementingClass, conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (ranges != null)
+ queryConfig.setRanges(ranges);
+
+ queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)).setUseIsolatedScanners(isIsolated(implementingClass, conf))
+ .setUseLocalIterators(usesLocalIterators(implementingClass, conf)).setOfflineScan(isOfflineScan(implementingClass, conf));
+ return Maps.immutableEntry(tableName, queryConfig);
+ }
+ return null;
+ }
+
+ public static Map<String,Map<KeyExtent,List<Range>>> binOffline(String tableId, List<Range> ranges, Instance instance, Connector conn)
+ throws AccumuloException, TableNotFoundException {
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ KeyExtent lastExtent = null;
+ while (rowIter.hasNext()) {
+ Iterator<Map.Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Map.Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily().equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
+ location = entry.getValue().toString();
+ }
+
+ if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+ return binnedRanges;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
index 0ba42cd,0000000..26680dc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
@@@ -1,204 -1,0 +1,205 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * @since 1.6.0
+ */
+public class OutputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link BatchWriter}.
+ *
+ * @since 1.6.0
+ */
+ public static enum WriteOpts {
+ DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ CAN_CREATE_TABLES, SIMULATION_MODE
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setDefaultTableName(Class<?> implementingClass, Configuration conf, String tableName) {
+ if (tableName != null)
+ conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the default table name
+ * @since 1.6.0
+ * @see #setDefaultTableName(Class, Configuration, String)
+ */
+ public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.6.0
+ */
+ public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf, BatchWriterConfig bwConfig) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String serialized;
+ try {
+ bwConfig.write(new DataOutputStream(baos));
- serialized = new String(baos.toByteArray(), Constants.UTF8);
++ serialized = new String(baos.toByteArray(), UTF_8);
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the configuration object
+ * @since 1.6.0
+ * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
+ */
+ public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
+ String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
+ BatchWriterConfig bwConfig = new BatchWriterConfig();
+ if (serialized == null || serialized.isEmpty()) {
+ return bwConfig;
+ } else {
+ try {
- ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(Constants.UTF8));
++ ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
+ bwConfig.readFields(new DataInputStream(bais));
+ bais.close();
+ return bwConfig;
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ }
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setCreateTables(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.6.0
+ * @see #setCreateTables(Class, Configuration, boolean)
+ */
+ public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setSimulationMode(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setSimulationMode(Class, Configuration, boolean)
+ */
+ public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 1541fae,0753282..9e26672
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@@ -26,11 -28,10 +28,10 @@@ import java.util.Arrays
import java.util.Scanner;
import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
-import org.apache.commons.codec.binary.Base64;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.DistributedCacheHelper;
+import org.apache.accumulo.core.util.Base64;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
index 0d3aca2,d95a647..95fcf53
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
@@@ -16,20 -16,18 +16,21 @@@
*/
package org.apache.accumulo.core.client.mock;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.io.ByteArrayInputStream;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Writer;
-import jline.ConsoleReader;
+import org.apache.commons.io.output.WriterOutputStream;
-import org.apache.accumulo.core.util.shell.Shell;
import org.apache.commons.cli.CommandLine;
+import jline.console.ConsoleReader;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.core.util.shell.ShellOptionsJC;
/**
* An Accumulo Shell implementation that allows a developer to attach an InputStream and Writer to the Shell for testing purposes.
@@@ -38,29 -36,12 +39,29 @@@ public class MockShell extends Shell
private static final String NEWLINE = "\n";
protected InputStream in;
- protected Writer writer;
-
- public MockShell(InputStream in, Writer writer) throws IOException {
+ protected OutputStream out;
+
+ /**
+ * Will only be set if you use either the Writer constructor or the setWriter(Writer) method
+ * @deprecated since 1.6.0; use out
+ */
+ @Deprecated
+ protected Writer writer = null;
+
+ public MockShell(InputStream in, OutputStream out) throws IOException {
super();
this.in = in;
- this.writer = writer;
+ this.out = out;
+ // we presume they don't use the writer field unless they use the other constructor.
+ }
+
+ /**
+ * @deprecated since 1.6.0; use OutputStream version
+ */
+ @Deprecated
+ public MockShell(InputStream in, Writer out) throws IOException {
- this(in, new WriterOutputStream(out, Constants.UTF8.name()));
++ this(in, new WriterOutputStream(out, UTF_8.name()));
+ this.writer = out;
}
public boolean config(String... args) {
@@@ -107,7 -79,7 +108,7 @@@
printInfo();
if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
- java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++ java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
try {
while (scanner.hasNextLine() && !hasExited()) {
execCommand(scanner.nextLine(), true, isVerbose());
@@@ -146,20 -118,11 +147,20 @@@
}
/**
- * @param writer
- * the writer to set
+ * @param out
+ * the output stream to set
+ */
+ public void setConsoleWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ /**
+ * @deprecated since 1.6.0; use the OutputStream version
*/
- public void setConsoleWriter(Writer writer) {
- this.writer = writer;
+ @Deprecated
+ public void setConsoleWriter(Writer out) {
- setConsoleWriter(new WriterOutputStream(out, Constants.UTF8.name()));
++ setConsoleWriter(new WriterOutputStream(out, UTF_8.name()));
+ this.writer = out;
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index bd9feba,05aa0c3..4d73979
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@@ -124,14 -123,10 +125,14 @@@ public class PasswordToken implements A
throw new RuntimeException(e);
}
}
-
- private void setPassword(CharBuffer charBuffer) {
+
+ protected void setPassword(byte[] password) {
+ this.password = Arrays.copyOf(password, password.length);
+ }
+
+ protected void setPassword(CharBuffer charBuffer) {
// encode() kicks back a C-string, which is not compatible with the old passwording system
- ByteBuffer bb = Constants.UTF8.encode(charBuffer);
+ ByteBuffer bb = UTF_8.encode(charBuffer);
// create array using byter buffer length
this.password = new byte[bb.remaining()];
bb.get(this.password);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 41a1f3b,0000000..618ff7e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -1,356 -1,0 +1,357 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.TreeMap;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.log4j.Logger;
+
+/**
+ * This class generates documentation to inform users of the available configuration properties in a presentable form.
+ */
+class ConfigurationDocGen {
+ private abstract class Format {
+
+ abstract void beginSection(String section);
+
+ void endSection() {}
+
+ void generate() {
+ pageHeader();
+
+ beginSection("Available Properties");
+ for (Property prefix : prefixes) {
+ if (!prefix.isExperimental()) {
+ prefixSection(prefix);
+ for (Property prop : sortedProps.values()) {
+ if (!prop.isExperimental()) {
+ property(prefix, prop);
+ }
+ }
+ }
+ }
+ endSection();
+
+ beginSection("Property Types");
+ propertyTypeDescriptions();
+ endSection();
+
+ pageFooter();
+ doc.close();
+ }
+
+ abstract String getExt();
+
+ void pageFooter() {}
+
+ // read static header content from resources and output
+ void pageHeader() {
+ appendResource("config-header." + getExt());
+ doc.println();
+ }
+
+ abstract void prefixSection(Property prefix);
+
+ abstract void property(Property prefix, Property prop);
+
+ abstract void propertyTypeDescriptions();
+
+ abstract String sanitize(String str);
+
+ }
+
+ private class HTML extends Format {
+ private boolean highlight;
+
+ private void beginRow() {
+ doc.println(startTag("tr", highlight ? "class='highlight'" : null));
+ highlight = !highlight;
+ }
+
+ @Override
+ void beginSection(String section) {
+ doc.println(t("h1", section, null));
+ highlight = true;
+ doc.println("<table>");
+ }
+
+ private String t(String tag, String cell, String options) {
+ return startTag(tag, options) + cell + "</" + tag + ">";
+ }
+
+ private void cellData(String cell, String options) {
+ doc.println(t("td", cell, options));
+ }
+
+ private void columnNames(String... names) {
+ beginRow();
+ for (String s : names)
+ doc.println(t("th", s, null));
+ endRow();
+ }
+
+ private void endRow() {
+ doc.println("</tr>");
+ }
+
+ @Override
+ void endSection() {
+ doc.println("</table>");
+ }
+
+ @Override
+ String getExt() {
+ return "html";
+ }
+
+ @Override
+ void pageFooter() {
+ doc.println("</body>");
+ doc.println("</html>");
+ }
+
+ @Override
+ void pageHeader() {
+ super.pageHeader();
+ doc.println("<p>Jump to: ");
+ String delimiter = "";
+ for (Property prefix : prefixes) {
+ if (!prefix.isExperimental()) {
+ doc.print(delimiter + "<a href='#" + prefix.name() + "'>" + prefix.getKey() + "*</a>");
+ delimiter = " | ";
+ }
+ }
+ doc.println("</p>");
+ }
+
+ @Override
+ void prefixSection(Property prefix) {
+ beginRow();
+ doc.println(t("td", t("span", prefix.getKey() + '*', "id='" + prefix.name() + "' class='large'"), "colspan='5'"
+ + (prefix.isDeprecated() ? " class='deprecated'" : "")));
+ endRow();
+ beginRow();
+ doc.println(t("td", (prefix.isDeprecated() ? t("b", t("i", "Deprecated. ", null), null) : "") + sanitize(prefix.getDescription()), "colspan='5'"
+ + (prefix.isDeprecated() ? " class='deprecated'" : "")));
+ endRow();
+
+ switch (prefix) {
+ case TABLE_CONSTRAINT_PREFIX:
+ break;
+ case TABLE_ITERATOR_PREFIX:
+ break;
+ case TABLE_LOCALITY_GROUP_PREFIX:
+ break;
+ case TABLE_COMPACTION_STRATEGY_PREFIX:
+ break;
+ default:
+ columnNames("Property", "Type", "ZooKeeper Mutable", "Default Value", "Description");
+ break;
+ }
+ }
+
+ @Override
+ void property(Property prefix, Property prop) {
+ boolean isDeprecated = prefix.isDeprecated() || prop.isDeprecated();
+ if (prop.getKey().startsWith(prefix.getKey())) {
+ beginRow();
+ cellData(prop.getKey(), isDeprecated ? "class='deprecated'" : null);
+ cellData("<b><a href='#" + prop.getType().name() + "'>" + prop.getType().toString().replaceAll(" ", " ") + "</a></b>",
+ isDeprecated ? "class='deprecated'" : null);
+ cellData(isZooKeeperMutable(prop), isDeprecated ? "class='deprecated'" : null);
+ cellData("<pre>" + (prop.getRawDefaultValue().isEmpty() ? " " : sanitize(prop.getRawDefaultValue().replaceAll(" ", " "))) + "</pre>",
+ isDeprecated ? "class='deprecated'" : null);
+ cellData((isDeprecated ? "<b><i>Deprecated.</i></b> " : "") + sanitize(prop.getDescription()), isDeprecated ? "class='deprecated'" : null);
+ endRow();
+ }
+
+ }
+
+ @Override
+ void propertyTypeDescriptions() {
+ columnNames("Property Type", "Description");
+ for (PropertyType type : PropertyType.values()) {
+ if (type == PropertyType.PREFIX)
+ continue;
+ beginRow();
+ cellData("<h3 id='" + type.name() + "'>" + type + "</h3>", null);
+ cellData(type.getFormatDescription(), null);
+ endRow();
+ }
+ }
+
+ @Override
+ String sanitize(String str) {
+ return str.replace("&", "&").replace("<", "<").replace(">", ">").replaceAll("(?:\r\n|\r|\n)", "<br />");
+ }
+
+ private String startTag(String tag, String options) {
+ return "<" + tag + (options != null ? " " + options : "") + ">";
+ }
+
+ }
+
+ private class LaTeX extends Format {
+ @Override
+ void beginSection(String section) {
+ doc.println("\\section{" + section + "}");
+ }
+
+ @Override
+ String getExt() {
+ return "tex";
+ }
+
+ @Override
+ void prefixSection(Property prefix) {
+ boolean depr = prefix.isDeprecated();
+ doc.println("\\subsection{" + prefix.getKey() + "*}" + (depr ? " (Deprecated)" : ""));
+ doc.println((depr ? "\\sout{\\textit{Deprecated.} " : "") + sanitize(prefix.getDescription()) + (depr ? "}" : ""));
+ doc.println();
+ }
+
+ @Override
+ void property(Property prefix, Property prop) {
+ boolean depr = prefix.isDeprecated() || prop.isDeprecated();
+ if (prop.getKey().startsWith(prefix.getKey())) {
+ doc.println("\\subsubsection{" + prop.getKey() + "}");
+ doc.println(strike((depr ? "\\textit{Deprecated.} " : "") + sanitize(prop.getDescription()), depr));
+ doc.println();
+ doc.println(strike("\\textit{Type:} " + prop.getType().name() + "\\\\", depr));
+ doc.println(strike("\\textit{ZooKeeper Mutable:} " + isZooKeeperMutable(prop) + "\\\\", depr));
+ doc.println(strike("\\textit{Default Value:} " + sanitize(prop.getRawDefaultValue()), depr));
+ doc.println();
+ }
+ }
+
+ private String strike(String s, boolean isDeprecated) {
+ return (isDeprecated ? "\\sout{" : "") + s + (isDeprecated ? "}" : "");
+ }
+
+ @Override
+ void propertyTypeDescriptions() {
+ for (PropertyType type : PropertyType.values()) {
+ if (type == PropertyType.PREFIX)
+ continue;
+ doc.println("\\subsection{" + sanitize(type.toString()) + "}");
+ doc.println(sanitize(type.getFormatDescription()));
+ doc.println();
+ }
+ }
+
+ @Override
+ String sanitize(String str) {
+ str = str.replace("\\", "\\textbackslash{}");
+ str = str.replace("~", "\\textasciitilde{}");
+ str = str.replace("^", "\\textasciicircum");
+
+ str = str.replace("&", "\\&");
+ str = str.replace("%", "\\%");
+ str = str.replace("$", "\\$");
+ str = str.replace("#", "\\#");
+ str = str.replace("_", "\\_");
+ str = str.replace("{", "\\{");
+ str = str.replace("}", "\\}");
+
+ str = str.replaceAll("(?:\r\n|\r|\n)", "\\\\\\\\\n");
+ return str;
+ }
+ }
+
+ private static Logger log = Logger.getLogger(ConfigurationDocGen.class);
+
+ private PrintStream doc;
+
+ private final ArrayList<Property> prefixes;
+
+ private final TreeMap<String,Property> sortedProps;
+
+ ConfigurationDocGen(PrintStream doc) {
+ this.doc = doc;
+ this.prefixes = new ArrayList<Property>();
+ this.sortedProps = new TreeMap<String,Property>();
+
+ for (Property prop : Property.values()) {
+ if (prop.isExperimental())
+ continue;
+
+ if (prop.getType() == PropertyType.PREFIX)
+ this.prefixes.add(prop);
+ else
+ this.sortedProps.put(prop.getKey(), prop);
+ }
+ }
+
+ private void appendResource(String resourceName) {
+ InputStream data = ConfigurationDocGen.class.getResourceAsStream(resourceName);
+ if (data != null) {
+ byte[] buffer = new byte[1024];
+ int n;
+ try {
+ while ((n = data.read(buffer)) > 0)
- doc.print(new String(buffer, 0, n, Constants.UTF8));
++ doc.print(new String(buffer, 0, n, UTF_8));
+ } catch (IOException e) {
+ e.printStackTrace();
+ return;
+ } finally {
+ try {
+ data.close();
+ } catch (IOException ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }
+
+ private String isZooKeeperMutable(Property prop) {
+ if (!Property.isValidZooPropertyKey(prop.getKey()))
+ return "no";
+ if (Property.isFixedZooPropertyKey(prop))
+ return "yes but requires restart of the " + prop.getKey().split("[.]")[0];
+ return "yes";
+ }
+
+ void generateHtml() {
+ new HTML().generate();
+ }
+
+ void generateLaTeX() {
+ new LaTeX().generate();
+ }
+
+ /*
+ * Generate documentation for conf/accumulo-site.xml file usage
+ */
+ public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
+ if (args.length == 2 && args[0].equals("--generate-html")) {
- new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateHtml();
++ new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateHtml();
+ } else if (args.length == 2 && args[0].equals("--generate-latex")) {
- new ConfigurationDocGen(new PrintStream(args[1], Constants.UTF8.name())).generateLaTeX();
++ new ConfigurationDocGen(new PrintStream(args[1], UTF_8.name())).generateLaTeX();
+ } else {
+ throw new IllegalArgumentException("Usage: " + ConfigurationDocGen.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index ca769cb,7c4f66e..437590f
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@@ -16,11 -16,10 +16,11 @@@
*/
package org.apache.accumulo.core.data;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.io.Serializable;
+import java.nio.ByteBuffer;
- import org.apache.accumulo.core.Constants;
-
public class ArrayByteSequence extends ByteSequence implements Serializable {
private static final long serialVersionUID = 1L;
@@@ -48,22 -47,9 +48,22 @@@
}
public ArrayByteSequence(String s) {
- this(s.getBytes(Constants.UTF8));
+ this(s.getBytes(UTF_8));
}
+ public ArrayByteSequence(ByteBuffer buffer) {
+ this.length = buffer.remaining();
+
+ if (buffer.hasArray()) {
+ this.data = buffer.array();
+ this.offset = buffer.position();
+ } else {
+ this.data = new byte[length];
+ this.offset = 0;
+ buffer.get(data);
+ }
+ }
+
@Override
public byte byteAt(int i) {
[18/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Condition.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Condition.java
index bfd4818,0000000..e6e31eb
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@@ -1,238 -1,0 +1,239 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Arrays;
+import java.util.HashSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Conditions that must be met on a particular column in a row.
+ *
+ * @since 1.6.0
+ */
+public class Condition {
+
+ private ByteSequence cf;
+ private ByteSequence cq;
+ private ByteSequence cv;
+ private ByteSequence val;
+ private Long ts;
+ private IteratorSetting iterators[] = new IteratorSetting[0];
+ private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+
+
+ public Condition(CharSequence cf, CharSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
- this.cf = new ArrayByteSequence(cf.toString().getBytes(Constants.UTF8));
- this.cq = new ArrayByteSequence(cq.toString().getBytes(Constants.UTF8));
++ this.cf = new ArrayByteSequence(cf.toString().getBytes(UTF_8));
++ this.cq = new ArrayByteSequence(cq.toString().getBytes(UTF_8));
+ this.cv = EMPTY;
+ }
+
+ public Condition(byte[] cf, byte[] cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf);
+ this.cq = new ArrayByteSequence(cq);
+ this.cv = EMPTY;
+ }
+
+ public Condition(Text cf, Text cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = new ArrayByteSequence(cf.getBytes(), 0, cf.getLength());
+ this.cq = new ArrayByteSequence(cq.getBytes(), 0, cq.getLength());
+ this.cv = EMPTY;
+ }
+
+ public Condition(ByteSequence cf, ByteSequence cq) {
+ ArgumentChecker.notNull(cf, cq);
+ this.cf = cf;
+ this.cq = cq;
+ this.cv = EMPTY;
+ }
+
+ public ByteSequence getFamily() {
+ return cf;
+ }
+
+ public ByteSequence getQualifier() {
+ return cq;
+ }
+
+ /**
+ * Sets the version for the column to check. If this is not set then the latest column will be checked, unless iterators do something different.
+ *
+ * @return returns this
+ */
+
+ public Condition setTimestamp(long ts) {
+ this.ts = ts;
+ return this;
+ }
+
+ public Long getTimestamp() {
+ return ts;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(CharSequence value) {
+ ArgumentChecker.notNull(value);
- this.val = new ArrayByteSequence(value.toString().getBytes(Constants.UTF8));
++ this.val = new ArrayByteSequence(value.toString().getBytes(UTF_8));
+ return this;
+ }
+
+ /**
+ * This method sets the expected value of a column. Inorder for the condition to pass the column must exist and have this value. If a value is not set, then
+ * the column must be absent for the condition to pass.
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(byte[] value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value);
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(Text value) {
+ ArgumentChecker.notNull(value);
+ this.val = new ArrayByteSequence(value.getBytes(), 0, value.getLength());
+ return this;
+ }
+
+ /**
+ * see {@link #setValue(byte[])}
+ *
+ * @return returns this
+ */
+
+ public Condition setValue(ByteSequence value) {
+ ArgumentChecker.notNull(value);
+ this.val = value;
+ return this;
+ }
+
+ public ByteSequence getValue() {
+ return val;
+ }
+
+ /**
+ * Sets the visibility for the column to check. If not set it defaults to empty visibility.
+ *
+ * @return returns this
+ */
+
+ public Condition setVisibility(ColumnVisibility cv) {
+ ArgumentChecker.notNull(cv);
+ this.cv = new ArrayByteSequence(cv.getExpression());
+ return this;
+ }
+
+ public ByteSequence getVisibility() {
+ return cv;
+ }
+
+ /**
+ * Set iterators to use when reading the columns value. These iterators will be applied in addition to the iterators configured for the table. Using iterators
+ * its possible to test other conditions, besides equality and absence, like less than. On the server side the iterators will be seeked using a range that
+ * covers only the family, qualifier, and visibility (if the timestamp is set then it will be used to narrow the range). Value equality will be tested using
+ * the first entry returned by the iterator stack.
+ *
+ * @return returns this
+ */
+
+ public Condition setIterators(IteratorSetting... iterators) {
+ ArgumentChecker.notNull(iterators);
+
+ if (iterators.length > 1) {
+ HashSet<String> names = new HashSet<String>();
+ HashSet<Integer> prios = new HashSet<Integer>();
+
+ for (IteratorSetting iteratorSetting : iterators) {
+ if (!names.add(iteratorSetting.getName()))
+ throw new IllegalArgumentException("iterator name used more than once " + iteratorSetting.getName());
+ if (!prios.add(iteratorSetting.getPriority()))
+ throw new IllegalArgumentException("iterator priority used more than once " + iteratorSetting.getPriority());
+ }
+ }
+
+ this.iterators = iterators;
+ return this;
+ }
+
+ public IteratorSetting[] getIterators() {
+ return iterators;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o == null || !(o instanceof Condition)) {
+ return false;
+ }
+ Condition c = (Condition) o;
+ if (!(c.cf.equals(cf))) {
+ return false;
+ }
+ if (!(c.cq.equals(cq))) {
+ return false;
+ }
+ if (!(c.cv.equals(cv))) {
+ return false;
+ }
+ if (!(c.val == null ? val == null : c.val.equals(val))) {
+ return false;
+ }
+ if (!(c.ts == null ? ts == null : c.ts.equals(ts))) {
+ return false;
+ }
+ if (!(Arrays.equals(c.iterators, iterators))) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 31 * result + cf.hashCode();
+ result = 31 * result + cq.hashCode();
+ result = 31 * result + cv.hashCode();
+ result = 31 * result + (val == null ? 0 : val.hashCode());
+ result = 31 * result + (ts == null ? 0 : ts.hashCode());
+ result = 31 * result + Arrays.hashCode(iterators);
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/data/Value.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/data/Value.java
index 11e60e1,bc1f71d..d256107
--- a/core/src/main/java/org/apache/accumulo/core/data/Value.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Value.java
@@@ -25,9 -26,6 +26,8 @@@ import java.io.IOException
import java.nio.ByteBuffer;
import java.util.List;
+import com.google.common.base.Preconditions;
+
- import org.apache.accumulo.core.Constants;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/IntersectingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
index 8006114,0b220ee..7073d11
--- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java
@@@ -24,10 -26,7 +26,9 @@@ import java.util.Collections
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.util.ArgumentChecker;
@@@ -195,13 -166,12 +196,13 @@@ public class Authorizations implements
}
/**
- * Retrieve a serialized form of the underlying set of authorizations.
+ * Returns a serialized form of these authorizations.
*
- * @see #Authorizations(byte[])
+ * @return serialized form of these authorizations, as a string encoded in UTF-8
+ * @see #serialize()
*/
public byte[] getAuthorizationsArray() {
- return serialize().getBytes(Constants.UTF8);
+ return serialize().getBytes(UTF_8);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index e76b0ef,66b68ab..e8f4e97
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -243,9 -223,9 +244,9 @@@ public class ColumnVisibility
/*
* Walks an expression's AST and appends a string representation to a supplied StringBuilder. This method adds parens where necessary.
*/
- private static void stringify(Node root, byte[] expression, StringBuilder out) {
+ public static void stringify(Node root, byte[] expression, StringBuilder out) {
if (root.type == NodeType.TERM) {
- out.append(new String(expression, root.start, root.end - root.start, Constants.UTF8));
+ out.append(new String(expression, root.start, root.end - root.start, UTF_8));
} else {
String sep = "";
for (Node c : root.children) {
@@@ -314,12 -293,12 +315,12 @@@
while (index < expression.length) {
switch (expression[index++]) {
case '&': {
- expr = processTerm(termStart, index - 1, expr, expression);
+ expr = processTerm(subtermStart, index - 1, expr, expression);
if (result != null) {
if (!result.type.equals(NodeType.AND))
- throw new BadArgumentException("cannot mix & and |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("cannot mix & and |", new String(expression, UTF_8), index - 1);
} else {
- result = new Node(NodeType.AND);
+ result = new Node(NodeType.AND, wholeTermStart);
}
result.add(expr);
expr = null;
@@@ -328,12 -307,12 +329,12 @@@
break;
}
case '|': {
- expr = processTerm(termStart, index - 1, expr, expression);
+ expr = processTerm(subtermStart, index - 1, expr, expression);
if (result != null) {
if (!result.type.equals(NodeType.OR))
- throw new BadArgumentException("cannot mix | and &", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("cannot mix | and &", new String(expression, UTF_8), index - 1);
} else {
- result = new Node(NodeType.OR);
+ result = new Node(NodeType.OR, wholeTermStart);
}
result.add(expr);
expr = null;
@@@ -343,18 -322,18 +344,18 @@@
}
case '(': {
parens++;
- if (termStart != index - 1 || expr != null)
+ if (subtermStart != index - 1 || expr != null)
- throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
expr = parse_(expression);
- termStart = index;
- termComplete = false;
+ subtermStart = index;
+ subtermComplete = false;
break;
}
case ')': {
parens--;
- Node child = processTerm(termStart, index - 1, expr, expression);
+ Node child = processTerm(subtermStart, index - 1, expr, expression);
if (child == null && result == null)
- throw new BadArgumentException("empty expression not allowed", new String(expression, Constants.UTF8), index);
+ throw new BadArgumentException("empty expression not allowed", new String(expression, UTF_8), index);
if (result == null)
return child;
if (result.type == child.type)
@@@ -366,8 -345,8 +367,8 @@@
return result;
}
case '"': {
- if (termStart != index - 1)
+ if (subtermStart != index - 1)
- throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
while (index < expression.length && expression[index] != '"') {
if (expression[index] == '\\') {
@@@ -379,20 -358,20 +380,20 @@@
}
if (index == expression.length)
- throw new BadArgumentException("unclosed quote", new String(expression, Constants.UTF8), subtermStart);
- throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), termStart);
-
- if (termStart + 1 == index)
- throw new BadArgumentException("empty term", new String(expression, UTF_8), termStart);
++ throw new BadArgumentException("unclosed quote", new String(expression, UTF_8), subtermStart);
+ if (subtermStart + 1 == index)
- throw new BadArgumentException("empty term", new String(expression, Constants.UTF8), subtermStart);
++ throw new BadArgumentException("empty term", new String(expression, UTF_8), subtermStart);
+
index++;
- termComplete = true;
+ subtermComplete = true;
break;
}
default: {
- if (termComplete)
+ if (subtermComplete)
- throw new BadArgumentException("expression needs & or |", new String(expression, Constants.UTF8), index - 1);
+ throw new BadArgumentException("expression needs & or |", new String(expression, UTF_8), index - 1);
byte c = expression[index - 1];
if (!Authorizations.isValidAuthChar(c))
@@@ -514,12 -482,10 +515,12 @@@
* .
* ColumnVisibility cv = new ColumnVisibility(quote("A#C") + "&" + quote("FOO"));
* </pre>
- *
+ *
+ * @param term term to quote
+ * @return quoted term (unquoted if unnecessary)
*/
public static String quote(String term) {
- return new String(quote(term.getBytes(Constants.UTF8)), Constants.UTF8);
+ return new String(quote(term.getBytes(UTF_8)), UTF_8);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/Credentials.java
index 582b4e0,31fe18d..0ecc31d
--- a/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/Credentials.java
@@@ -16,17 -16,10 +16,18 @@@
*/
package org.apache.accumulo.core.security;
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.nio.ByteBuffer;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.Base64;
/**
* A wrapper for internal use. This class carries the instance, principal, and authentication token for use in the public API, in a non-serialized form. This is
@@@ -53,102 -36,9 +54,102 @@@ public class Credentials
this.principal = principal;
this.token = token;
}
-
- public TCredentials toThrift() {
- return CredentialHelper.createSquelchError(principal, token, instance.getInstanceID());
+
+ /**
+ * Gets the principal.
+ *
+ * @return unique identifier for the entity (e.g. a user or service) authorized for these credentials
+ */
+ public String getPrincipal() {
+ return principal;
+ }
+
+ /**
+ * Gets the authentication token.
+ *
+ * @return authentication token used to prove that the principal for these credentials has been properly verified
+ */
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ /**
+ * Converts the current object to the relevant thrift type. The object returned from this contains a non-destroyable version of the
+ * {@link AuthenticationToken}, so this should be used just before placing on the wire, and references to it should be tightly controlled.
+ *
+ * @param instance
+ * client instance
+ * @return Thrift credentials
+ * @throws RuntimeException
+ * if the authentication token has been destroyed (expired)
+ */
+ public TCredentials toThrift(Instance instance) {
+ TCredentials tCreds = new TCredentials(getPrincipal(), getToken().getClass().getName(),
+ ByteBuffer.wrap(AuthenticationTokenSerializer.serialize(getToken())), instance.getInstanceID());
+ if (getToken().isDestroyed())
+ throw new RuntimeException("Token has been destroyed", new AccumuloSecurityException(getPrincipal(), SecurityErrorCode.TOKEN_EXPIRED));
+ return tCreds;
+ }
+
+ /**
+ * Converts a given thrift object to our internal Credentials representation.
+ * @param serialized a Thrift encoded set of credentials
+ * @return a new Credentials instance; destroy the token when you're done.
+ */
+ public static Credentials fromThrift(TCredentials serialized) {
+ return new Credentials(serialized.getPrincipal(), AuthenticationTokenSerializer.deserialize(serialized.getTokenClassName(), serialized.getToken()));
+ }
+
+ /**
+ * Converts the current object to a serialized form. The object returned from this contains a non-destroyable version of the {@link AuthenticationToken}, so
+ * references to it should be tightly controlled.
+ *
+ * @return serialized form of these credentials
+ */
+ public final String serialize() {
- return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(Constants.UTF8))) + ":"
- + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(Constants.UTF8))) + ":"
++ return (getPrincipal() == null ? "-" : Base64.encodeBase64String(getPrincipal().getBytes(UTF_8))) + ":"
++ + (getToken() == null ? "-" : Base64.encodeBase64String(getToken().getClass().getName().getBytes(UTF_8))) + ":"
+ + (getToken() == null ? "-" : Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(getToken())));
+ }
+
+ /**
+ * Converts the serialized form to an instance of {@link Credentials}. The original serialized form will not be affected.
+ *
+ * @param serializedForm
+ * serialized form of credentials
+ * @return deserialized credentials
+ */
+ public static final Credentials deserialize(String serializedForm) {
+ String[] split = serializedForm.split(":", 3);
- String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), Constants.UTF8);
- String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), Constants.UTF8);
++ String principal = split[0].equals("-") ? null : new String(Base64.decodeBase64(split[0]), UTF_8);
++ String tokenType = split[1].equals("-") ? null : new String(Base64.decodeBase64(split[1]), UTF_8);
+ AuthenticationToken token = null;
+ if (!split[2].equals("-")) {
+ byte[] tokenBytes = Base64.decodeBase64(split[2]);
+ token = AuthenticationTokenSerializer.deserialize(tokenType, tokenBytes);
+ }
+ return new Credentials(principal, token);
+ }
+
+ @Override
+ public int hashCode() {
+ return getPrincipal() == null ? 0 : getPrincipal().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof Credentials))
+ return false;
+ Credentials other = Credentials.class.cast(obj);
+ boolean pEq = getPrincipal() == null ? (other.getPrincipal() == null) : (getPrincipal().equals(other.getPrincipal()));
+ if (!pEq)
+ return false;
+ boolean tEq = getToken() == null ? (other.getToken() == null) : (getToken().equals(other.getToken()));
+ return tEq;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getName() + ":" + getPrincipal() + ":" + (getToken() == null ? null : getToken().getClass().getName()) + ":<hidden>";
}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityConstraint.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
index be5f008,9c5782b..178de91
--- a/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/VisibilityParseException.java
@@@ -16,27 -16,17 +16,27 @@@
*/
package org.apache.accumulo.core.security;
- import java.text.ParseException;
+ import static com.google.common.base.Charsets.UTF_8;
- import org.apache.accumulo.core.Constants;
+ import java.text.ParseException;
+/**
+ * An exception thrown when a visibility string cannot be parsed.
+ */
public class VisibilityParseException extends ParseException {
private static final long serialVersionUID = 1L;
private String visibility;
-
+
+ /**
+ * Creates a new exception.
+ *
+ * @param reason reason string
+ * @param visibility visibility that could not be parsed
+ * @param errorOffset offset into visibility where parsing failed
+ */
public VisibilityParseException(String reason, byte[] visibility, int errorOffset) {
super(reason, errorOffset);
- this.visibility = new String(visibility, Constants.UTF8);
+ this.visibility = new String(visibility, UTF_8);
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
index 7a23c37,f6a6a01..8771832
--- a/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ByteBufferUtil.java
@@@ -22,8 -24,6 +24,7 @@@ import java.util.Arrays
import java.util.Collection;
import java.util.List;
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.ByteSequence;
import org.apache.hadoop.io.Text;
public class ByteBufferUtil {
@@@ -62,18 -62,6 +63,18 @@@
}
public static String toString(ByteBuffer bytes) {
- return new String(bytes.array(), bytes.position(), bytes.remaining(), Constants.UTF8);
+ return new String(bytes.array(), bytes.position(), bytes.remaining(), UTF_8);
}
+
+ public static ByteBuffer toByteBuffers(ByteSequence bs) {
+ if (bs == null)
+ return null;
+
+ if (bs.isBackedByArray()) {
+ return ByteBuffer.wrap(bs.getBackingArray(), bs.offset(), bs.length());
+ } else {
+ // TODO create more efficient impl
+ return ByteBuffer.wrap(bs.toArray());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index 9f86db3,0000000..26a8c73
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@@ -1,117 -1,0 +1,118 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import jline.console.ConsoleReader;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOpts.Password;
+import org.apache.accumulo.core.cli.ClientOpts.PasswordConverter;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.Base64;
+
+import com.beust.jcommander.Parameter;
+
+public class CreateToken {
+
+ private static ConsoleReader reader = null;
+
+ private static ConsoleReader getConsoleReader() throws IOException {
+ if (reader == null)
+ reader = new ConsoleReader();
+ return reader;
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-u", "--user"}, description = "Connection user")
+ public String principal = null;
+
+ @Parameter(names = "-p", converter = PasswordConverter.class, description = "Connection password")
+ public Password password = null;
+
+ @Parameter(names = "--password", converter = PasswordConverter.class, description = "Enter the connection password", password = true)
+ public Password securePassword = null;
+
+ @Parameter(names = {"-tc", "--tokenClass"}, description = "The class of the authentication token")
+ public String tokenClassName = PasswordToken.class.getName();
+
+ @Parameter(names = {"-f", "--file"}, description = "The filename to save the auth token to. Multiple tokens can be stored in the same file,"
+ + " but only the first for each user will be recognized.")
+ public String tokenFile = null;
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(CreateToken.class.getName(), args);
+
+ Password pass = opts.password;
+ if (pass == null && opts.securePassword != null) {
+ pass = opts.securePassword;
+ }
+
+ try {
+ String principal = opts.principal;
+ if (principal == null) {
+ principal = getConsoleReader().readLine("Username (aka principal): ");
+ }
+
+ AuthenticationToken token = Class.forName(opts.tokenClassName).asSubclass(AuthenticationToken.class).newInstance();
+ Properties props = new Properties();
+ for (TokenProperty tp : token.getProperties()) {
+ String input;
+ if (pass != null && tp.getKey().equals("password")) {
+ input = pass.toString();
+ } else {
+ if (tp.getMask()) {
+ input = getConsoleReader().readLine(tp.getDescription() + ": ", '*');
+ } else {
+ input = getConsoleReader().readLine(tp.getDescription() + ": ");
+ }
+ }
+ props.put(tp.getKey(), input);
+ token.init(props);
+ }
+ String tokenBase64 = Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token));
+
+ String tokenFile = opts.tokenFile;
+ if (tokenFile == null) {
+ tokenFile = getConsoleReader().readLine("File to save auth token to: ");
+ }
+ File tf = new File(tokenFile);
+ if (!tf.exists()) {
+ tf.createNewFile();
+ }
- PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, Constants.UTF8.name());
++ PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, UTF_8.name());
+ String outString = principal + ":" + opts.tokenClassName + ":" + tokenBase64;
+ out.println(outString);
+ out.close();
+ System.out.println("Token written to " + tokenFile + ". Remember to upload it to hdfs.");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Encoding.java
index aff8f62,9999c3c..b26b363
--- a/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Encoding.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.core.util;
- import org.apache.accumulo.core.Constants;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import org.apache.commons.codec.binary.Base64;
+import org.apache.accumulo.core.util.Base64;
import org.apache.hadoop.io.Text;
public class Encoding {
@@@ -36,8 -38,10 +37,8 @@@
public static byte[] decodeBase64FileName(String node) {
while (node.length() % 4 != 0)
node += "=";
-
- node = node.replace('_', '/').replace('-', '+');
-
+ /* decode transparently handles URLSafe encodings */
- return Base64.decodeBase64(node.getBytes(Constants.UTF8));
+ return Base64.decodeBase64(node.getBytes(UTF_8));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java
index 02c8be3,3830108..05067b5
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@@ -21,7 -23,7 +23,6 @@@ import java.util.Iterator
import java.util.List;
import java.util.Map.Entry;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
@@@ -236,8 -233,8 +237,8 @@@ public class Merge
while (iterator.hasNext()) {
Entry<Key,Value> entry = iterator.next();
Key key = entry.getKey();
- if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- String[] sizeEntries = new String(entry.getValue().get(), Constants.UTF8).split(",");
+ String[] sizeEntries = new String(entry.getValue().get(), UTF_8).split(",");
if (sizeEntries.length == 2) {
tabletSize += Long.parseLong(sizeEntries[0]);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
index 1ebbf13,0000000..e71a377
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/MonitorUtil.java
@@@ -1,31 -1,0 +1,33 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.zookeeper.KeeperException;
+
+public class MonitorUtil {
+ public static String getLocation(Instance instance) throws KeeperException, InterruptedException {
+ ZooReader zr = new ZooReader(instance.getZooKeepers(), 5000);
+ byte[] loc = zr.getData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, null);
- return loc==null ? null : new String(loc, Constants.UTF8);
++ return loc==null ? null : new String(loc, UTF_8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index da5648c,755bee2..b187a76
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -557,14 -437,11 +559,14 @@@ public class Shell extends ShellOption
} catch (IOException e) {
log.warn("Unable to load history file at " + historyPath);
}
-
+
+ // Turn Ctrl+C into Exception instead of JVM exit
+ reader.setHandleUserInterrupt(true);
+
ShellCompletor userCompletor = null;
-
+
if (execFile != null) {
- java.util.Scanner scanner = new java.util.Scanner(execFile, Constants.UTF8.name());
- java.util.Scanner scanner = new java.util.Scanner(new File(execFile), UTF_8.name());
++ java.util.Scanner scanner = new java.util.Scanner(execFile, UTF_8.name());
try {
while (scanner.hasNextLine() && !hasExited()) {
execCommand(scanner.nextLine(), true, isVerbose());
@@@ -974,14 -805,14 +976,14 @@@
@Override
public void close() {}
};
-
+
public static class PrintFile implements PrintLine {
PrintWriter writer;
-
+
public PrintFile(String filename) throws FileNotFoundException {
- writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), Constants.UTF8)));
+ writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename), UTF_8)));
}
-
+
@Override
public void print(String s) {
writer.println(s);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
index f06a639,c325612..90a3548
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AddSplitsCommand.java
@@@ -19,9 -21,7 +21,8 @@@ import static com.google.common.base.Ch
import java.io.File;
import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.util.Base64;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
import org.apache.commons.cli.CommandLine;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
index bcb6c24,5c86ac9..44dce0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/AuthenticateCommand.java
@@@ -35,12 -36,12 +36,12 @@@ public class AuthenticateCommand extend
final String user = cl.getArgs()[0];
final String p = shellState.readMaskedLine("Enter current password for '" + user + "': ", '*');
if (p == null) {
- shellState.getReader().printNewline();
+ shellState.getReader().println();
return 0;
} // user canceled
- final byte[] password = p.getBytes(Constants.UTF8);
+ final byte[] password = p.getBytes(UTF_8);
final boolean valid = shellState.getConnector().securityOperations().authenticateUser(user, new PasswordToken(password));
- shellState.getReader().printString((valid ? "V" : "Not v") + "alid\n");
+ shellState.getReader().println((valid ? "V" : "Not v") + "alid");
return 0;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
index d2c73f7,8e39115..fc1af0f
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
@@@ -16,16 -16,16 +16,17 @@@
*/
package org.apache.accumulo.core.util.shell.commands;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.io.File;
import java.io.IOException;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableExistsException;
@@@ -73,12 -70,12 +74,12 @@@ public class CreateTableCommand extend
}
final SortedSet<Text> partitions = new TreeSet<Text>();
final boolean decode = cl.hasOption(base64Opt.getOpt());
-
+
if (cl.hasOption(createTableOptSplit.getOpt())) {
final String f = cl.getOptionValue(createTableOptSplit.getOpt());
-
+
String line;
- Scanner file = new Scanner(new File(f), Constants.UTF8.name());
+ Scanner file = new Scanner(new File(f), UTF_8.name());
try {
while (file.hasNextLine()) {
line = file.nextLine();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
index 61f60f8,4f1c793..06d36b9
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/HiddenCommand.java
@@@ -19,8 -21,6 +21,7 @@@ import static com.google.common.base.Ch
import java.security.SecureRandom;
import java.util.Random;
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.Base64;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
import org.apache.accumulo.core.util.shell.ShellCommandException;
@@@ -39,11 -40,12 +40,11 @@@ public class HiddenCommand extends Comm
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
if (rand.nextInt(10) == 0) {
shellState.getReader().beep();
- shellState.getReader().printNewline();
- shellState.getReader().printString(
+ shellState.getReader().println();
+ shellState.getReader().println(
new String(Base64.decodeBase64(("ICAgICAgIC4tLS4KICAgICAgLyAvXCBcCiAgICAgKCAvLS1cICkKICAgICAuPl8gIF88LgogICAgLyB8ICd8ICcgXAog"
+ "ICAvICB8Xy58Xy4gIFwKICAvIC98ICAgICAgfFwgXAogfCB8IHwgfFwvfCB8IHwgfAogfF98IHwgfCAgfCB8IHxffAogICAgIC8gIF9fICBcCiAgICAvICAv"
- + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(Constants.UTF8)), Constants.UTF8));
+ + "ICBcICBcCiAgIC8gIC8gICAgXCAgXF8KIHwvICAvICAgICAgXCB8IHwKIHxfXy8gICAgICAgIFx8X3wK").getBytes(UTF_8)), UTF_8));
- shellState.getReader().printNewline();
} else {
throw new ShellCommandException(ErrorCode.UNRECOGNIZED_COMMAND, getName());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/PasswdCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
index a0743f4,01fb92e..b8fc4c4
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/UserCommand.java
@@@ -41,10 -42,10 +42,10 @@@ public class UserCommand extends Comman
// of these methods fails
final String p = shellState.readMaskedLine("Enter password for user " + user + ": ", '*');
if (p == null) {
- shellState.getReader().printNewline();
+ shellState.getReader().println();
return 0;
} // user canceled
- pass = p.getBytes(Constants.UTF8);
+ pass = p.getBytes(UTF_8);
shellState.updateUser(user, new PasswordToken(pass));
return 0;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
index 259cb06,3794f29..c276056
--- a/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/BatchWriterConfigTest.java
@@@ -16,6 -16,8 +16,7 @@@
*/
package org.apache.accumulo.core.client;
+ import static com.google.common.base.Charsets.UTF_8;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
index dde5575,0000000..dce742f
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/ZooKeeperInstanceTest.java
@@@ -1,157 -1,0 +1,158 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ZooKeeperInstanceTest {
+ private static final UUID IID = UUID.randomUUID();
+ private static final String IID_STRING = IID.toString();
+ private ClientConfiguration config;
+ private ZooCacheFactory zcf;
+ private ZooCache zc;
+ private ZooKeeperInstance zki;
+
+ private void mockIdConstruction(ClientConfiguration config) {
+ expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
+ expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
+ expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
+ }
+
+ private void mockNameConstruction(ClientConfiguration config) {
+ expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(null);
+ expect(config.get(ClientProperty.INSTANCE_NAME)).andReturn("instance");
+ expect(config.get(ClientProperty.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(config.get(ClientProperty.INSTANCE_ZK_TIMEOUT)).andReturn("30");
+ }
+
+ @Before
+ public void setUp() {
+ config = createMock(ClientConfiguration.class);
+ mockNameConstruction(config);
+ replay(config);
+ zcf = createMock(ZooCacheFactory.class);
+ zc = createMock(ZooCache.class);
+ expect(zcf.getZooCache("zk1", 30000)).andReturn(zc).anyTimes();
+ replay(zcf);
+ zki = new ZooKeeperInstance(config, zcf);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidConstruction() {
+ config = createMock(ClientConfiguration.class);
+ expect(config.get(ClientProperty.INSTANCE_ID)).andReturn(IID_STRING);
+ mockNameConstruction(config);
+ replay(config);
+ new ZooKeeperInstance(config);
+ }
+
+ @Test
+ public void testSimpleGetters() {
+ assertEquals("instance", zki.getInstanceName());
+ assertEquals("zk1", zki.getZooKeepers());
+ assertEquals(30000, zki.getZooKeepersSessionTimeOut());
+ }
+
+ @Test
+ public void testGetInstanceID_FromCache() {
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
+ replay(zc);
+ assertEquals(IID_STRING, zki.getInstanceID());
+ }
+
+ @Test
+ public void testGetInstanceID_Direct() {
+ config = createMock(ClientConfiguration.class);
+ mockIdConstruction(config);
+ replay(config);
+ zki = new ZooKeeperInstance(config, zcf);
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
+ replay(zc);
+ assertEquals(IID_STRING, zki.getInstanceID());
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetInstanceID_NoMapping() {
+ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(null);
+ replay(zc);
+ zki.getInstanceID();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetInstanceID_IDMissingForName() {
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/instance")).andReturn(IID_STRING.getBytes(UTF_8));
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
+ replay(zc);
+ zki.getInstanceID();
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetInstanceID_IDMissingForID() {
+ config = createMock(ClientConfiguration.class);
+ mockIdConstruction(config);
+ replay(config);
+ zki = new ZooKeeperInstance(config, zcf);
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn(null);
+ replay(zc);
+ zki.getInstanceID();
+ }
+
+ @Test
+ public void testGetInstanceName() {
+ config = createMock(ClientConfiguration.class);
+ mockIdConstruction(config);
+ replay(config);
+ zki = new ZooKeeperInstance(config, zcf);
+ expect(zc.get(Constants.ZROOT + "/" + IID_STRING)).andReturn("yup".getBytes());
+ List<String> children = new java.util.ArrayList<String>();
+ children.add("child1");
+ children.add("child2");
+ expect(zc.getChildren(Constants.ZROOT + Constants.ZINSTANCES)).andReturn(children);
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(Constants.UTF8));
- expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(Constants.UTF8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child1")).andReturn(UUID.randomUUID().toString().getBytes(UTF_8));
++ expect(zc.get(Constants.ZROOT + Constants.ZINSTANCES + "/child2")).andReturn(IID_STRING.getBytes(UTF_8));
+ replay(zc);
+ assertEquals("child2", zki.getInstanceName());
+ }
+
+ @Test
+ public void testAllZooKeepersAreUsed() {
+ final String zookeepers = "zk1,zk2,zk3", instanceName = "accumulo";
+ ZooCacheFactory factory = createMock(ZooCacheFactory.class);
+ expect(factory.getZooCache(zookeepers, 30000)).andReturn(zc).anyTimes();
+ replay(factory);
+ ClientConfiguration cfg = ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers);
+ ZooKeeperInstance zki = new ZooKeeperInstance(cfg, factory);
+ assertEquals(zookeepers, zki.getZooKeepers());
+ assertEquals(instanceName, zki.getInstanceName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index db4a12b,0000000..e2661e2
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@@ -1,306 -1,0 +1,306 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client.mock;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Random;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Filter;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.Test;
+
+public class MockNamespacesTest {
+
+ Random random = new Random();
+
+ /**
+ * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
+ */
+ @Test
+ public void testDefaultNamespace() throws Exception {
+ String tableName = "test";
+ Instance instance = new MockInstance("default");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE));
+ c.tableOperations().create(tableName);
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ /**
+ * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2"
+ * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the
+ * tables and delete the namespace.
+ */
+ @Test
+ public void testCreateAndDeleteNamespace() throws Exception {
+ String namespace = "testing";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ Instance instance = new MockInstance("createdelete");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().create(tableName1);
+ assertTrue(c.tableOperations().exists(tableName1));
+
+ c.tableOperations().create(tableName2);
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ // deleting
+ try {
+ // can't delete a namespace with tables in it
+ c.namespaceOperations().delete(namespace);
+ fail();
+ } catch (NamespaceNotEmptyException e) {
+ // ignore, supposed to happen
+ }
+ assertTrue(c.namespaceOperations().exists(namespace));
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(c.tableOperations().exists(tableName2));
+
+ c.tableOperations().delete(tableName2);
+ assertTrue(!c.tableOperations().exists(tableName2));
+ assertTrue(c.namespaceOperations().exists(namespace));
+
+ c.tableOperations().delete(tableName1);
+ assertTrue(!c.tableOperations().exists(tableName1));
+ c.namespaceOperations().delete(namespace);
+ assertTrue(!c.namespaceOperations().exists(namespace));
+ }
+
+ /**
+ * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a
+ * namespace-wide level, use {@link NamespaceOperations}.
+ *
+ * Checks to make sure namespace-level properties are overridden by table-level properties.
+ *
+ * Checks to see if the default namespace's properties work as well.
+ */
+
+ @Test
+ public void testNamespaceProperties() throws Exception {
+ String namespace = "propchange";
+ String tableName1 = namespace + ".table1";
+ String tableName2 = namespace + ".table2";
+
+ String propKey = Property.TABLE_SCAN_MAXMEM.getKey();
+ String propVal = "42K";
+
+ Instance instance = new MockInstance("props");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName1);
+ c.namespaceOperations().setProperty(namespace, propKey, propVal);
+
+ // check the namespace has the property
+ assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal));
+
+ // check that the table gets it from the namespace
+ assertTrue(checkTableHasProp(c, tableName1, propKey, propVal));
+
+ // test a second table to be sure the first wasn't magical
+ // (also, changed the order, the namespace has the property already)
+ c.tableOperations().create(tableName2);
+ assertTrue(checkTableHasProp(c, tableName2, propKey, propVal));
+
+ // test that table properties override namespace properties
+ String propKey2 = Property.TABLE_FILE_MAX.getKey();
+ String propVal2 = "42";
+ String tablePropVal = "13";
+
+ c.tableOperations().setProperty(tableName2, propKey2, tablePropVal);
+ c.namespaceOperations().setProperty("propchange", propKey2, propVal2);
+
+ assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal));
+
+ // now check that you can change the default namespace's properties
+ propVal = "13K";
+ String tableName = "some_table";
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal);
+
+ assertTrue(checkTableHasProp(c, tableName, propKey, propVal));
+
+ // test the properties server-side by configuring an iterator.
+ // should not show anything with column-family = 'a'
+ String tableName3 = namespace + ".table3";
+ c.tableOperations().create(tableName3);
+
+ IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName());
+ c.namespaceOperations().attachIterator(namespace, setting);
+
+ BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
+ m.put("a", "b", new Value("abcde".getBytes()));
+ bw.addMutation(m);
+ bw.flush();
+ bw.close();
+
+ // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY);
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+ }
+
+ /**
+ * This test renames and clones two separate table into different namespaces. different namespace.
+ */
+ @Test
+ public void testRenameAndCloneTableToNewNamespace() throws Exception {
+ String namespace1 = "renamed";
+ String namespace2 = "cloned";
+ String tableName = "table";
+ String tableName1 = "renamed.table1";
+ // String tableName2 = "cloned.table2";
+
+ Instance instance = new MockInstance("renameclone");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.tableOperations().create(tableName);
+ c.namespaceOperations().create(namespace1);
+ c.namespaceOperations().create(namespace2);
+
+ c.tableOperations().rename(tableName, tableName1);
+
+ assertTrue(c.tableOperations().exists(tableName1));
+ assertTrue(!c.tableOperations().exists(tableName));
+
+ // TODO implement clone in mock
+ /*
+ * c.tableOperations().clone(tableName1, tableName2, false, null, null);
+ *
+ * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2));
+ */
+ return;
+ }
+
+ /**
+ * This test renames a namespace and ensures that its tables are still correct
+ */
+ @Test
+ public void testNamespaceRename() throws Exception {
+ String namespace1 = "n1";
+ String namespace2 = "n2";
+ String table = "t";
+
+ Instance instance = new MockInstance("rename");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ c.namespaceOperations().create(namespace1);
+ c.tableOperations().create(namespace1 + "." + table);
+
+ c.namespaceOperations().rename(namespace1, namespace2);
+
+ assertTrue(!c.namespaceOperations().exists(namespace1));
+ assertTrue(c.namespaceOperations().exists(namespace2));
+ assertTrue(!c.tableOperations().exists(namespace1 + "." + table));
+ assertTrue(c.tableOperations().exists(namespace2 + "." + table));
+ }
+
+ /**
+ * This tests adding iterators to a namespace, listing them, and removing them
+ */
+ @Test
+ public void testNamespaceIterators() throws Exception {
+ Instance instance = new MockInstance("Iterators");
+ Connector c = instance.getConnector("user", new PasswordToken("pass"));
+
+ String namespace = "iterator";
+ String tableName = namespace + ".table";
+ String iter = "thing";
+
+ c.namespaceOperations().create(namespace);
+ c.tableOperations().create(tableName);
+
+ IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName());
+ HashSet<IteratorScope> scope = new HashSet<IteratorScope>();
+ scope.add(IteratorScope.scan);
+ c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope));
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("r");
- m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8)));
++ m.put("a", "b", new Value("abcde".getBytes(UTF_8)));
+ bw.addMutation(m);
+ bw.flush();
+
+ Scanner s = c.createScanner(tableName, Authorizations.EMPTY);
+ System.out.println(s.iterator().next());
+ // do scanners work correctly in mock?
+ // assertTrue(!s.iterator().hasNext());
+
+ assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter));
+ c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope));
+ }
+
+ private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException {
+ for (Entry<String,String> e : c.tableOperations().getProperties(t)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException,
+ AccumuloSecurityException {
+ for (Entry<String,String> e : c.namespaceOperations().getProperties(n)) {
+ if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static class SimpleFilter extends Filter {
+ @Override
+ public boolean accept(Key k, Value v) {
+ if (k.getColumnFamily().toString().equals("a"))
+ return false;
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
index 17d2297,0000000..7bb3fa4
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderTokenTest.java
@@@ -1,129 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.security.tokens;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CredentialProviderTokenTest {
+
+ private static boolean isCredentialProviderAvailable = false;
+
+ // Keystore contains: {'root.password':'password', 'bob.password':'bob'}
+ private static String keystorePath;
+
+ @BeforeClass
+ public static void setup() {
+ try {
+ Class.forName(CredentialProviderFactoryShim.HADOOP_CRED_PROVIDER_CLASS_NAME);
+ isCredentialProviderAvailable = true;
+ } catch (Exception e) {
+ isCredentialProviderAvailable = false;
+ }
+
+ URL keystoreUrl = CredentialProviderTokenTest.class.getResource("/passwords.jceks");
+ Assert.assertNotNull(keystoreUrl);
+ keystorePath = "jceks://file/" + new File(keystoreUrl.getFile()).getAbsolutePath();
+ }
+
+ @Test
+ public void testPasswordsFromCredentialProvider() throws Exception {
+ if (!isCredentialProviderAvailable) {
+ return;
+ }
+
+ CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
- Assert.assertArrayEquals("password".getBytes(Constants.UTF8), token.getPassword());
++ Assert.assertArrayEquals("password".getBytes(UTF_8), token.getPassword());
+
+ token = new CredentialProviderToken("bob.password", keystorePath);
- Assert.assertArrayEquals("bob".getBytes(Constants.UTF8), token.getPassword());
++ Assert.assertArrayEquals("bob".getBytes(UTF_8), token.getPassword());
+ }
+
+ @Test
+ public void testEqualityAfterInit() throws Exception {
+ if (!isCredentialProviderAvailable) {
+ return;
+ }
+
+ CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
+
+ CredentialProviderToken uninitializedToken = new CredentialProviderToken();
+ Properties props = new Properties();
+ props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
+ props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
+ uninitializedToken.init(props);
+
+ Assert.assertArrayEquals(token.getPassword(), uninitializedToken.getPassword());
+ }
+
+ @Test
+ public void testMissingClassesThrowsException() throws Exception {
+ if (isCredentialProviderAvailable) {
+ return;
+ }
+
+ try {
+ new CredentialProviderToken("root.password", keystorePath);
+ Assert.fail("Should fail to create CredentialProviderToken when classes are not available");
+ } catch (IOException e) {
+ // pass
+ }
+ }
+
+ @Test
+ public void cloneReturnsCorrectObject() throws Exception {
+ if (!isCredentialProviderAvailable) {
+ return;
+ }
+
+ CredentialProviderToken token = new CredentialProviderToken("root.password", keystorePath);
+ CredentialProviderToken clone = token.clone();
+
+ Assert.assertEquals(token, clone);
+ Assert.assertArrayEquals(token.getPassword(), clone.getPassword());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingProperties() throws Exception {
+ CredentialProviderToken token = new CredentialProviderToken();
+ token.init(new Properties());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingNameProperty() throws Exception {
+ CredentialProviderToken token = new CredentialProviderToken();
+ Properties props = new Properties();
+ props.put(CredentialProviderToken.NAME_PROPERTY, "root.password");
+ token.init(props);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missingProviderProperty() throws Exception {
+ CredentialProviderToken token = new CredentialProviderToken();
+ Properties props = new Properties();
+ props.put(CredentialProviderToken.CREDENTIAL_PROVIDERS_PROPERTY, keystorePath);
+ token.init(props);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
index 3570610,0000000..d9a47d7
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionTest.java
@@@ -1,251 -1,0 +1,251 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConditionTest {
+ private static final ByteSequence EMPTY = new ArrayByteSequence(new byte[0]);
+ private static final String FAMILY = "family";
+ private static final String QUALIFIER = "qualifier";
+ private static final String VISIBILITY = "visibility";
+ private static final String VALUE = "value";
+ private static final IteratorSetting[] ITERATORS = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
+ new IteratorSetting(3, "third", "yetanotherclass")};
+
+ private String toString(ByteSequence bs) {
+ if (bs == null) {
+ return null;
+ }
- return new String(bs.toArray(), Constants.UTF8);
++ return new String(bs.toArray(), UTF_8);
+ }
+
+ private Condition c;
+
+ @Before
+ public void setUp() throws Exception {
+ c = new Condition(FAMILY, QUALIFIER);
+ }
+
+ @Test
+ public void testConstruction_CharSequence() {
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testConstruction_ByteArray() {
- c = new Condition(FAMILY.getBytes(Constants.UTF8), QUALIFIER.getBytes(Constants.UTF8));
++ c = new Condition(FAMILY.getBytes(UTF_8), QUALIFIER.getBytes(UTF_8));
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testConstruction_Text() {
+ c = new Condition(new Text(FAMILY), new Text(QUALIFIER));
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testConstruction_ByteSequence() {
- c = new Condition(new ArrayByteSequence(FAMILY.getBytes(Constants.UTF8)), new ArrayByteSequence(QUALIFIER.getBytes(Constants.UTF8)));
++ c = new Condition(new ArrayByteSequence(FAMILY.getBytes(UTF_8)), new ArrayByteSequence(QUALIFIER.getBytes(UTF_8)));
+ assertEquals(FAMILY, toString(c.getFamily()));
+ assertEquals(QUALIFIER, toString(c.getQualifier()));
+ assertEquals(EMPTY, c.getVisibility());
+ }
+
+ @Test
+ public void testGetSetTimestamp() {
+ c.setTimestamp(1234L);
+ assertEquals(Long.valueOf(1234L), c.getTimestamp());
+ }
+
+ @Test
+ public void testSetValue_CharSequence() {
+ c.setValue(VALUE);
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testSetValue_ByteArray() {
- c.setValue(VALUE.getBytes(Constants.UTF8));
++ c.setValue(VALUE.getBytes(UTF_8));
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testSetValue_Text() {
+ c.setValue(new Text(VALUE));
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testSetValue_ByteSequence() {
- c.setValue(new ArrayByteSequence(VALUE.getBytes(Constants.UTF8)));
++ c.setValue(new ArrayByteSequence(VALUE.getBytes(UTF_8)));
+ assertEquals(VALUE, toString(c.getValue()));
+ }
+
+ @Test
+ public void testGetSetVisibility() {
+ ColumnVisibility vis = new ColumnVisibility(VISIBILITY);
+ c.setVisibility(vis);
+ assertEquals(VISIBILITY, toString(c.getVisibility()));
+ }
+
+ @Test
+ public void testGetSetIterators() {
+ c.setIterators(ITERATORS);
+ assertArrayEquals(ITERATORS, c.getIterators());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetIterators_DuplicateName() {
+ IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
+ new IteratorSetting(3, "first", "yetanotherclass")};
+ c.setIterators(iterators);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetIterators_DuplicatePriority() {
+ IteratorSetting[] iterators = {new IteratorSetting(1, "first", "someclass"), new IteratorSetting(2, "second", "someotherclass"),
+ new IteratorSetting(1, "third", "yetanotherclass")};
+ c.setIterators(iterators);
+ }
+
+ @Test
+ public void testEquals() {
+ ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
+ c.setVisibility(cvis);
+ c.setValue(VALUE);
+ c.setTimestamp(1234L);
+ c.setIterators(ITERATORS);
+
+ // reflexivity
+ assertTrue(c.equals(c));
+
+ // non-nullity
+ assertFalse(c.equals(null));
+
+ // symmetry
+ Condition c2 = new Condition(FAMILY, QUALIFIER);
+ c2.setVisibility(cvis);
+ c2.setValue(VALUE);
+ c2.setTimestamp(1234L);
+ c2.setIterators(ITERATORS);
+ assertTrue(c.equals(c2));
+ assertTrue(c2.equals(c));
+
+ Condition c3 = new Condition("nope", QUALIFIER);
+ c3.setVisibility(cvis);
+ c3.setValue(VALUE);
+ c3.setTimestamp(1234L);
+ c3.setIterators(ITERATORS);
+ assertFalse(c.equals(c3));
+ assertFalse(c3.equals(c));
+ c3 = new Condition(FAMILY, "nope");
+ c3.setVisibility(cvis);
+ c3.setValue(VALUE);
+ c3.setTimestamp(1234L);
+ c3.setIterators(ITERATORS);
+ assertFalse(c.equals(c3));
+ assertFalse(c3.equals(c));
+
+ c2.setVisibility(new ColumnVisibility("sekrit"));
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setVisibility(cvis);
+ c2.setValue(EMPTY);
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setValue(VALUE);
+ c2.setTimestamp(2345L);
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setTimestamp(1234L);
+ c2.setIterators(new IteratorSetting[0]);
+ assertFalse(c.equals(c2));
+ assertFalse(c2.equals(c));
+ c2.setIterators(ITERATORS);
+ assertTrue(c.equals(c2));
+ assertTrue(c2.equals(c));
+
+ // set everything but vis, so its null
+ Condition c4 = new Condition(FAMILY, QUALIFIER);
+ c4.setValue(VALUE);
+ c4.setTimestamp(1234L);
+ c4.setIterators(ITERATORS);
+
+ assertFalse(c.equals(c4));
+ assertFalse(c4.equals(c));
+
+ // set everything but timestamp, so its null
+ Condition c5 = new Condition(FAMILY, QUALIFIER);
+ c5.setVisibility(cvis);
+ c5.setValue(VALUE);
+ c5.setIterators(ITERATORS);
+
+ assertFalse(c.equals(c5));
+ assertFalse(c5.equals(c));
+
+ // set everything but value
+ Condition c6 = new Condition(FAMILY, QUALIFIER);
+ c6.setVisibility(cvis);
+ c6.setTimestamp(1234L);
+ c6.setIterators(ITERATORS);
+
+ assertFalse(c.equals(c6));
+ assertFalse(c6.equals(c));
+
+ // test w/ no optional fields set
+ Condition c7 = new Condition(FAMILY, QUALIFIER);
+ Condition c8 = new Condition(FAMILY, QUALIFIER);
+ assertTrue(c7.equals(c8));
+ assertTrue(c8.equals(c7));
+
+ }
+
+ @Test
+ public void testHashCode() {
+ ColumnVisibility cvis = new ColumnVisibility(VISIBILITY);
+ c.setVisibility(cvis);
+ c.setValue(VALUE);
+ c.setTimestamp(1234L);
+ c.setIterators(ITERATORS);
+ int hc1 = c.hashCode();
+
+ Condition c2 = new Condition(FAMILY, QUALIFIER);
+ c2.setVisibility(cvis);
+ c2.setValue(VALUE);
+ c2.setTimestamp(1234L);
+ c2.setIterators(ITERATORS);
+ assertTrue(c.equals(c2));
+ assertEquals(hc1, c2.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 47f44f1,0000000..908c391
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@@ -1,149 -1,0 +1,149 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.data;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConditionalMutationTest {
- private static final byte[] ROW = "row".getBytes(Constants.UTF8);
++ private static final byte[] ROW = "row".getBytes(UTF_8);
+ private static final String FAMILY = "family";
+ private static final String QUALIFIER = "qualifier";
+ private static final String QUALIFIER2 = "qualifier2";
+ private static final String QUALIFIER3 = "qualifier3";
+
+ private Condition c1, c2;
+ private ConditionalMutation cm;
+
+ @Before
+ public void setUp() throws Exception {
+ c1 = new Condition(FAMILY, QUALIFIER);
+ c2 = new Condition(FAMILY, QUALIFIER2);
+ assertFalse(c1.equals(c2));
+ cm = new ConditionalMutation(ROW, c1, c2);
+ }
+
+ @Test
+ public void testConstruction_ByteArray() {
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_ByteArray_StartAndLength() {
+ cm = new ConditionalMutation(ROW, 1, 1, c1, c2);
- assertArrayEquals("o".getBytes(Constants.UTF8), cm.getRow());
++ assertArrayEquals("o".getBytes(UTF_8), cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_Text() {
+ cm = new ConditionalMutation(new Text(ROW), c1, c2);
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_CharSequence() {
- cm = new ConditionalMutation(new String(ROW, Constants.UTF8), c1, c2);
++ cm = new ConditionalMutation(new String(ROW, UTF_8), c1, c2);
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testConstruction_ByteSequence() {
+ cm = new ConditionalMutation(new ArrayByteSequence(ROW), c1, c2);
+ assertArrayEquals(ROW, cm.getRow());
+ List<Condition> cs = cm.getConditions();
+ assertEquals(2, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ }
+
+ @Test
+ public void testCopyConstructor() {
+ ConditionalMutation cm2 = new ConditionalMutation(cm);
+ assertArrayEquals(cm.getRow(), cm2.getRow());
+ assertEquals(cm.getConditions(), cm2.getConditions());
+ }
+
+ @Test
+ public void testAddCondition() {
+ Condition c3 = new Condition(FAMILY, QUALIFIER3);
+ cm.addCondition(c3);
+ List<Condition> cs = cm.getConditions();
+ assertEquals(3, cs.size());
+ assertEquals(c1, cs.get(0));
+ assertEquals(c2, cs.get(1));
+ assertEquals(c3, cs.get(2));
+ }
+
+ @Test
+ public void testEquals() {
+ // reflexivity
+ assertTrue(cm.equals(cm));
+
+ // non-nullity
+ assertFalse(cm.equals((Object) null));
+
+ // symmetry
+ ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
+ assertTrue(cm.equals(cm2));
+ assertTrue(cm2.equals(cm));
+
- ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(Constants.UTF8), c1, c2);
++ ConditionalMutation cm3 = new ConditionalMutation("row2".getBytes(UTF_8), c1, c2);
+ assertFalse(cm.equals(cm3));
+ cm3 = new ConditionalMutation(ROW, c2, c1);
+ assertFalse(cm.getConditions().equals(cm3.getConditions()));
+ assertFalse(cm.equals(cm3));
+ }
+
+ @Test
+ public void testEquals_Mutation() {
+ Mutation m = new Mutation(ROW);
+ assertFalse(m.equals(cm));
+ assertFalse(cm.equals(m));
+ }
+
+ @Test
+ public void testHashcode() {
+ ConditionalMutation cm2 = new ConditionalMutation(ROW, c1, c2);
+ assertTrue(cm.equals(cm2));
+ assertEquals(cm2.hashCode(), cm.hashCode());
+ }
+}
[21/50] [abbrv] git commit: Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
Merge branch '1.5' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9b20a9d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9b20a9d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9b20a9d4
Branch: refs/heads/master
Commit: 9b20a9d4062f9ac0d34ce4b2acd7b9b72524a84f
Parents: 5e0f6a4 2deabd3
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Oct 31 23:37:27 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Oct 31 23:37:27 2014 -0400
----------------------------------------------------------------------
assemble/bin/bootstrap_config.sh | 2 +-
.../org/apache/accumulo/core/Constants.java | 13 +++++-
.../apache/accumulo/core/cli/ClientOpts.java | 6 ++-
.../accumulo/core/client/BatchWriterConfig.java | 11 ++---
.../accumulo/core/client/ZooKeeperInstance.java | 14 ++++---
.../client/impl/InstanceOperationsImpl.java | 4 +-
.../accumulo/core/client/impl/Namespaces.java | 6 ++-
.../accumulo/core/client/impl/ServerClient.java | 4 +-
.../core/client/impl/TableOperationsImpl.java | 32 ++++++++-------
.../accumulo/core/client/impl/Tables.java | 10 +++--
.../accumulo/core/client/impl/Writer.java | 5 ++-
.../core/client/lexicoder/StringLexicoder.java | 6 +--
.../core/client/mapreduce/RangeInputSplit.java | 7 ++--
.../mapreduce/lib/impl/ConfiguratorBase.java | 4 +-
.../mapreduce/lib/impl/InputConfigurator.java | 8 ++--
.../mapreduce/lib/impl/OutputConfigurator.java | 7 ++--
.../lib/partition/RangePartitioner.java | 7 ++--
.../accumulo/core/client/mock/MockShell.java | 11 ++---
.../client/security/tokens/PasswordToken.java | 5 ++-
.../accumulo/core/conf/ConfigurationDocGen.java | 9 ++--
.../accumulo/core/data/ArrayByteSequence.java | 8 ++--
.../org/apache/accumulo/core/data/Column.java | 6 +--
.../apache/accumulo/core/data/Condition.java | 9 ++--
.../org/apache/accumulo/core/data/Value.java | 4 +-
.../accumulo/core/file/BloomFilterLayer.java | 7 ++--
.../core/iterators/ColumnFamilyCounter.java | 5 ++-
.../accumulo/core/iterators/LongCombiner.java | 7 ++--
.../accumulo/core/iterators/conf/ColumnSet.java | 7 ++--
.../core/iterators/user/BigDecimalCombiner.java | 7 ++--
.../core/iterators/user/GrepIterator.java | 5 ++-
.../iterators/user/IntersectingIterator.java | 7 ++--
.../core/iterators/user/LargeRowFilter.java | 5 ++-
.../core/iterators/user/RegExFilter.java | 7 ++--
.../iterators/user/RowDeletingIterator.java | 5 ++-
.../iterators/user/SummingArrayCombiner.java | 7 ++--
.../iterators/user/TransformingIterator.java | 7 ++--
.../core/iterators/user/VisibilityFilter.java | 5 ++-
.../accumulo/core/security/Authorizations.java | 13 +++---
.../core/security/ColumnVisibility.java | 43 ++++++++++----------
.../accumulo/core/security/Credentials.java | 11 ++---
.../core/security/VisibilityConstraint.java | 5 ++-
.../core/security/VisibilityParseException.java | 6 +--
.../core/trace/InstanceUserPassword.java | 5 ++-
.../accumulo/core/trace/ZooTraceClient.java | 5 ++-
.../core/util/ByteArrayBackedCharSequence.java | 5 ++-
.../apache/accumulo/core/util/ByteArraySet.java | 6 +--
.../accumulo/core/util/ByteBufferUtil.java | 5 ++-
.../apache/accumulo/core/util/CreateToken.java | 5 ++-
.../org/apache/accumulo/core/util/Encoding.java | 5 ++-
.../apache/accumulo/core/util/FastFormat.java | 4 +-
.../org/apache/accumulo/core/util/Merge.java | 5 ++-
.../apache/accumulo/core/util/MonitorUtil.java | 4 +-
.../org/apache/accumulo/core/util/TextUtil.java | 4 +-
.../apache/accumulo/core/util/shell/Shell.java | 6 ++-
.../util/shell/commands/AddSplitsCommand.java | 7 ++--
.../shell/commands/AuthenticateCommand.java | 5 ++-
.../util/shell/commands/CreateTableCommand.java | 7 ++--
.../util/shell/commands/ExecfileCommand.java | 5 ++-
.../core/util/shell/commands/HiddenCommand.java | 5 ++-
.../core/util/shell/commands/PasswdCommand.java | 5 ++-
.../shell/commands/QuotedStringTokenizer.java | 5 ++-
.../core/util/shell/commands/UserCommand.java | 5 ++-
.../core/client/BatchWriterConfigTest.java | 6 +--
.../core/client/ZooKeeperInstanceTest.java | 9 ++--
.../core/client/mock/MockNamespacesTest.java | 4 +-
.../tokens/CredentialProviderTokenTest.java | 7 ++--
.../security/tokens/PasswordTokenTest.java | 7 ++--
.../accumulo/core/data/ConditionTest.java | 12 +++---
.../core/data/ConditionalMutationTest.java | 10 ++---
.../apache/accumulo/core/data/ValueTest.java | 5 ++-
.../core/iterators/user/RegExFilterTest.java | 7 ++--
.../core/security/ColumnVisibilityTest.java | 6 +--
.../accumulo/core/security/CredentialsTest.java | 6 +--
.../core/security/VisibilityConstraintTest.java | 4 +-
.../security/crypto/BlockedIOStreamTest.java | 10 ++---
.../core/util/format/DeleterFormatterTest.java | 12 +++---
.../ShardedTableDistributionFormatterTest.java | 8 ++--
.../org/apache/accumulo/fate/AdminUtil.java | 8 ++--
.../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++----
.../zookeeper/DistributedReadWriteLock.java | 18 ++++----
.../accumulo/fate/zookeeper/ZooCache.java | 8 ++--
.../apache/accumulo/fate/zookeeper/ZooLock.java | 6 +--
.../accumulo/fate/zookeeper/ZooReservation.java | 9 ++--
.../accumulo/fate/zookeeper/ZooSession.java | 6 +--
.../impl/MiniAccumuloClusterImpl.java | 4 +-
.../org/apache/accumulo/proxy/ProxyServer.java | 7 ++--
.../apache/accumulo/proxy/TestProxyClient.java | 15 +++----
.../java/org/apache/accumulo/proxy/Util.java | 5 ++-
.../org/apache/accumulo/server/Accumulo.java | 4 +-
.../accumulo/server/client/HdfsZooInstance.java | 10 +++--
.../server/conf/ZooCachePropertyAccessor.java | 5 ++-
.../accumulo/server/conf/ZooConfiguration.java | 4 +-
.../server/constraints/MetadataConstraints.java | 14 ++++---
.../apache/accumulo/server/init/Initialize.java | 15 +++----
.../accumulo/server/master/LiveTServerSet.java | 3 +-
.../server/master/state/DeadServerList.java | 7 ++--
.../server/master/state/TServerInstance.java | 7 ++--
.../master/state/TabletStateChangeIterator.java | 5 ++-
.../accumulo/server/master/state/ZooStore.java | 5 ++-
.../master/state/ZooTabletStateStore.java | 11 ++---
.../server/metrics/AbstractMetricsImpl.java | 5 ++-
.../accumulo/server/monitor/LogService.java | 4 +-
.../server/security/SecurityOperation.java | 4 +-
.../server/security/SystemCredentials.java | 10 +++--
.../security/handler/ZKAuthenticator.java | 4 +-
.../server/security/handler/ZKAuthorizor.java | 5 ++-
.../server/security/handler/ZKPermHandler.java | 5 ++-
.../accumulo/server/tables/TableManager.java | 18 ++++----
.../server/tablets/UniqueNameAllocator.java | 10 +++--
.../server/util/AddFilesWithMissingEntries.java | 5 ++-
.../accumulo/server/util/ChangeSecret.java | 5 ++-
.../accumulo/server/util/CleanZookeeper.java | 6 ++-
.../accumulo/server/util/DeleteZooInstance.java | 6 ++-
.../accumulo/server/util/DumpZookeeper.java | 5 ++-
.../accumulo/server/util/ListInstances.java | 6 ++-
.../accumulo/server/util/MetadataTableUtil.java | 32 ++++++++-------
.../accumulo/server/util/NamespacePropUtil.java | 4 +-
.../accumulo/server/util/RandomizeVolumes.java | 6 +--
.../accumulo/server/util/RestoreZookeeper.java | 7 ++--
.../accumulo/server/util/SendLogToChainsaw.java | 9 ++--
.../accumulo/server/util/SystemPropUtil.java | 4 +-
.../accumulo/server/util/TablePropUtil.java | 4 +-
.../accumulo/server/util/TabletServerLocks.java | 4 +-
.../server/watcher/MonitorLog4jWatcher.java | 4 +-
.../accumulo/server/zookeeper/ZooQueueLock.java | 9 ++--
.../server/zookeeper/ZooReaderWriter.java | 5 ++-
.../zookeeper/ZooReaderWriterFactory.java | 5 ++-
.../server/conf/NamespaceConfigurationTest.java | 10 +++--
.../conf/ServerConfigurationFactoryTest.java | 4 +-
.../server/conf/TableConfigurationTest.java | 9 ++--
.../conf/ZooCachePropertyAccessorTest.java | 4 +-
.../java/org/apache/accumulo/master/Master.java | 8 ++--
.../master/recovery/RecoveryManager.java | 4 +-
.../accumulo/master/state/SetGoalState.java | 4 +-
.../accumulo/master/tableOps/BulkImport.java | 8 ++--
.../master/tableOps/CancelCompactions.java | 10 +++--
.../accumulo/master/tableOps/CompactRange.java | 14 ++++---
.../accumulo/master/tableOps/ExportTable.java | 8 ++--
.../accumulo/master/tableOps/ImportTable.java | 16 ++++----
.../accumulo/master/tableOps/RenameTable.java | 6 ++-
.../apache/accumulo/master/tableOps/Utils.java | 14 ++++---
.../master/tserverOps/ShutdownTServer.java | 4 +-
.../org/apache/accumulo/monitor/Monitor.java | 6 ++-
.../accumulo/monitor/ZooKeeperStatus.java | 7 ++--
.../accumulo/monitor/servlets/BasicServlet.java | 10 +++--
.../monitor/servlets/DefaultServlet.java | 5 ++-
.../accumulo/monitor/servlets/ShellServlet.java | 5 ++-
.../accumulo/monitor/servlets/XMLServlet.java | 7 ++--
.../accumulo/monitor/servlets/trace/Basic.java | 5 ++-
.../org/apache/accumulo/tracer/TraceServer.java | 8 ++--
.../tserver/BulkFailedCopyProcessor.java | 5 ++-
.../org/apache/accumulo/tserver/Tablet.java | 10 +++--
.../apache/accumulo/tserver/TabletServer.java | 9 ++--
.../apache/accumulo/tserver/log/DfsLogger.java | 4 +-
.../accumulo/tserver/logger/LogFileValue.java | 11 ++---
.../accumulo/tserver/logger/LogReader.java | 5 ++-
.../accumulo/test/BulkImportDirectory.java | 5 ++-
.../accumulo/test/NativeMapConcurrencyTest.java | 5 ++-
.../accumulo/test/NativeMapPerformanceTest.java | 7 ++--
.../accumulo/test/NativeMapStressTest.java | 5 ++-
.../apache/accumulo/test/TestBinaryRows.java | 7 ++--
.../org/apache/accumulo/test/TestIngest.java | 7 ++--
.../accumulo/test/TestMultiTableIngest.java | 5 ++-
.../test/continuous/ContinuousIngest.java | 8 ++--
.../test/continuous/ContinuousMoru.java | 7 ++--
.../test/continuous/ContinuousQuery.java | 5 ++-
.../test/continuous/ContinuousScanner.java | 5 ++-
.../test/continuous/ContinuousVerify.java | 7 ++--
.../test/continuous/ContinuousWalk.java | 17 ++++----
.../accumulo/test/continuous/Histogram.java | 6 +--
.../test/continuous/PrintScanTimeHistogram.java | 5 ++-
.../accumulo/test/continuous/TimeBinner.java | 5 ++-
.../test/continuous/UndefinedAnalyzer.java | 9 ++--
.../test/functional/CacheTestReader.java | 7 ++--
.../test/functional/CacheTestWriter.java | 11 ++---
.../accumulo/test/functional/ZombieTServer.java | 4 +-
.../metadata/MetadataBatchScanTest.java | 7 ++--
.../apache/accumulo/test/randomwalk/Module.java | 7 ++--
.../test/randomwalk/bulk/BulkMinusOne.java | 5 ++-
.../accumulo/test/randomwalk/bulk/Verify.java | 5 ++-
.../test/randomwalk/concurrent/BatchWrite.java | 5 ++-
.../test/randomwalk/concurrent/BulkImport.java | 5 ++-
.../concurrent/ChangeAuthorizations.java | 7 ++--
.../randomwalk/concurrent/StopTabletServer.java | 4 +-
.../accumulo/test/randomwalk/image/Verify.java | 5 ++-
.../accumulo/test/randomwalk/image/Write.java | 9 ++--
.../test/randomwalk/multitable/Write.java | 5 ++-
.../randomwalk/security/SecurityHelper.java | 7 ++--
.../test/randomwalk/security/TableOp.java | 9 ++--
.../test/randomwalk/sequential/Write.java | 5 ++-
.../test/randomwalk/shard/BulkInsert.java | 5 ++-
.../test/randomwalk/shard/ExportIndex.java | 5 ++-
.../accumulo/test/scalability/Ingest.java | 5 ++-
.../apache/accumulo/test/AuditMessageIT.java | 6 +--
.../org/apache/accumulo/test/NamespacesIT.java | 8 ++--
.../java/org/apache/accumulo/test/VolumeIT.java | 6 +--
.../accumulo/test/functional/AddSplitIT.java | 5 ++-
.../test/functional/BadIteratorMincIT.java | 6 +--
.../test/functional/BatchScanSplitIT.java | 7 ++--
.../test/functional/BatchWriterFlushIT.java | 5 ++-
.../accumulo/test/functional/BulkFileIT.java | 5 ++-
.../accumulo/test/functional/ConcurrencyIT.java | 7 ++--
.../accumulo/test/functional/ConstraintIT.java | 23 ++++++-----
.../test/functional/CreateAndUseIT.java | 5 ++-
.../test/functional/DeleteEverythingIT.java | 4 +-
.../test/functional/DeleteRowsSplitIT.java | 6 +--
.../accumulo/test/functional/LargeRowIT.java | 7 ++--
.../accumulo/test/functional/NativeMapIT.java | 15 +++----
.../test/functional/ScanIteratorIT.java | 7 ++--
.../accumulo/test/functional/ScanRangeIT.java | 5 ++-
.../test/functional/ScanSessionTimeOutIT.java | 5 ++-
.../test/functional/SplitRecoveryIT.java | 3 +-
.../accumulo/test/functional/TabletIT.java | 4 +-
.../accumulo/test/functional/VisibilityIT.java | 11 ++---
.../instrument/receivers/ZooSpanClient.java | 4 +-
215 files changed, 900 insertions(+), 667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/assemble/bin/bootstrap_config.sh
----------------------------------------------------------------------
diff --cc assemble/bin/bootstrap_config.sh
index fb643da,0000000..fd262f0
mode 100755,000000..100755
--- a/assemble/bin/bootstrap_config.sh
+++ b/assemble/bin/bootstrap_config.sh
@@@ -1,323 -1,0 +1,323 @@@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function usage {
+ cat <<EOF
+Usage: bootstrap_config.sh [-options]
+where options include (long options not available on all platforms):
+ -d, --dir Alternate directory to setup config files
+ -s, --size Supported sizes: '1GB' '2GB' '3GB' '512MB'
+ -n, --native Configure to use native libraries
+ -j, --jvm Configure to use the jvm
+ -o, --overwrite Overwrite the default config directory
+ -v, --version Specify the Apache Hadoop version supported versions: '1' '2'
+ -h, --help Print this help message
+EOF
+}
+
+# Start: Resolve Script Directory
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+# Stop: Resolve Script Directory
+
+#
+# Resolve accumulo home for bootstrapping
+#
+ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd )
+TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates"
+CONF_DIR="${ACCUMULO_HOME}/conf"
+ACCUMULO_SITE=accumulo-site.xml
+ACCUMULO_ENV=accumulo-env.sh
+
+SIZE=
+TYPE=
+HADOOP_VERSION=
+OVERWRITE="0"
+BASE_DIR=
+
+#Execute getopt
+if [[ $(uname -s) == "Linux" ]]; then
+ args=$(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@")
+else # Darwin, BSD
+ args=$(getopt b:d:s:njov:h $*)
+fi
+
+#Bad arguments
+if [[ $? != 0 ]]; then
+ usage 1>&2
+ exit 1
+fi
+eval set -- $args
+
+for i
+do
+ case "$i" in
+ -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers
+ BASE_DIR=$2; shift
+ shift;;
+ -d|--dir)
+ CONF_DIR=$2; shift
+ shift;;
+ -s|--size)
+ SIZE=$2; shift
+ shift;;
+ -n|--native)
+ TYPE=native
+ shift;;
+ -j|--jvm)
+ TYPE=jvm
+ shift;;
+ -o|--overwrite)
+ OVERWRITE=1
+ shift;;
+ -v|--version)
+ HADOOP_VERSION=$2; shift
+ shift;;
+ -h|--help)
+ usage
+ exit 0
+ shift;;
+ --)
+ shift
+ break;;
+ esac
+done
+
+while [[ "${OVERWRITE}" = "0" ]]; do
+ if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then
+ echo "Warning your current config files in ${CONF_DIR} will be overwritten!"
+ echo
+ echo "How would you like to proceed?:"
+ select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do
+ if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then
+ echo -n "Please specifiy new conf directory: "
+ read CONF_DIR
+ elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then
+ OVERWRITE=1
+ fi
+ break
+ done
+ else
+ OVERWRITE=1
+ fi
+done
- echo "Coppying configuration files to: ${CONF_DIR}"
++echo "Copying configuration files to: ${CONF_DIR}"
+
+#Native 1GB
+native_1GB_tServer="-Xmx128m -Xms128m"
+_1GB_master="-Xmx128m -Xms128m"
+_1GB_monitor="-Xmx64m -Xms64m"
+_1GB_gc="-Xmx64m -Xms64m"
+_1GB_other="-Xmx128m -Xms64m"
+
+_1GB_memoryMapMax="256M"
+native_1GB_nativeEnabled="true"
+_1GB_cacheDataSize="15M"
+_1GB_cacheIndexSize="40M"
+_1GB_sortBufferSize="50M"
+_1GB_waLogMaxSize="256M"
+
+#Native 2GB
+native_2GB_tServer="-Xmx256m -Xms256m"
+_2GB_master="-Xmx256m -Xms256m"
+_2GB_monitor="-Xmx128m -Xms64m"
+_2GB_gc="-Xmx128m -Xms128m"
+_2GB_other="-Xmx256m -Xms64m"
+
+_2GB_memoryMapMax="512M"
+native_2GB_nativeEnabled="true"
+_2GB_cacheDataSize="30M"
+_2GB_cacheIndexSize="80M"
+_2GB_sortBufferSize="50M"
+_2GB_waLogMaxSize="512M"
+
+#Native 3GB
+native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m"
+_3GB_master="-Xmx1g -Xms1g"
+_3GB_monitor="-Xmx1g -Xms256m"
+_3GB_gc="-Xmx256m -Xms256m"
+_3GB_other="-Xmx1g -Xms256m"
+
+_3GB_memoryMapMax="1G"
+native_3GB_nativeEnabled="true"
+_3GB_cacheDataSize="128M"
+_3GB_cacheIndexSize="128M"
+_3GB_sortBufferSize="200M"
+_3GB_waLogMaxSize="1G"
+
+#Native 512MB
+native_512MB_tServer="-Xmx48m -Xms48m"
+_512MB_master="-Xmx128m -Xms128m"
+_512MB_monitor="-Xmx64m -Xms64m"
+_512MB_gc="-Xmx64m -Xms64m"
+_512MB_other="-Xmx128m -Xms64m"
+
+_512MB_memoryMapMax="80M"
+native_512MB_nativeEnabled="true"
+_512MB_cacheDataSize="7M"
+_512MB_cacheIndexSize="20M"
+_512MB_sortBufferSize="50M"
+_512MB_waLogMaxSize="100M"
+
+#JVM 1GB
+jvm_1GB_tServer="-Xmx384m -Xms384m"
+
+jvm_1GB_nativeEnabled="false"
+
+#JVM 2GB
+jvm_2GB_tServer="-Xmx768m -Xms768m"
+
+jvm_2GB_nativeEnabled="false"
+
+#JVM 3GB
+jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G"
+
+jvm_3GB_nativeEnabled="false"
+
+#JVM 512MB
+jvm_512MB_tServer="-Xmx128m -Xms128m"
+
+jvm_512MB_nativeEnabled="false"
+
+
+if [[ -z "${SIZE}" ]]; then
+ echo "Choose the heap configuration:"
+ select DIRNAME in 1GB 2GB 3GB 512MB; do
+ echo "Using '${DIRNAME}' configuration"
+ SIZE=${DIRNAME}
+ break
+ done
+elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB" && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then
+ echo "Invalid memory size"
+ echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'"
+ exit 1
+fi
+
+if [[ -z "${TYPE}" ]]; then
+ echo
+ echo "Choose the Accumulo memory-map type:"
+ select TYPENAME in Java Native; do
+ if [[ "${TYPENAME}" == "Native" ]]; then
+ TYPE="native"
+ echo "Don't forget to build the native libraries using the bin/build_native_library.sh script"
+ elif [[ "${TYPENAME}" == "Java" ]]; then
+ TYPE="jvm"
+ fi
+ echo "Using '${TYPE}' configuration"
+ echo
+ break
+ done
+fi
+
+if [[ -z "${HADOOP_VERSION}" ]]; then
+ echo
+ echo "Choose the Apache Hadoop version:"
+ select HADOOP in 'HADOOP 1' 'HADOOP 2' ; do
+ if [ "${HADOOP}" == "HADOOP 2" ]; then
+ HADOOP_VERSION="2"
+ elif [ "${HADOOP}" == "HADOOP 1" ]; then
+ HADOOP_VERSION="1"
+ fi
+ echo "Using Apache Hadoop version '${HADOOP_VERSION}' configuration"
+ echo
+ break
+ done
+elif [[ "${HADOOP_VERSION}" != "1" && "${HADOOP_VERSION}" != "2" ]]; then
+ echo "Invalid Apache Hadoop version"
+ echo "Supported Apache Hadoop versions: '1' '2'"
+ exit 1
+fi
+
+for var in SIZE TYPE HADOOP_VERSION; do
+ if [[ -z ${!var} ]]; then
+ echo "Invalid $var configuration"
+ exit 1
+ fi
+done
+
+TSERVER="${TYPE}_${SIZE}_tServer"
+MASTER="_${SIZE}_master"
+MONITOR="_${SIZE}_monitor"
+GC="_${SIZE}_gc"
+OTHER="_${SIZE}_other"
+
+MEMORY_MAP_MAX="_${SIZE}_memoryMapMax"
+NATIVE="${TYPE}_${SIZE}_nativeEnabled"
+CACHE_DATA_SIZE="_${SIZE}_cacheDataSize"
+CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize"
+SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize"
+WAL_MAX_SIZE="_${SIZE}_waLogMaxSize"
+
+MAVEN_PROJ_BASEDIR=""
+
+if [[ ! -z "${BASE_DIR}" ]]; then
+ MAVEN_PROJ_BASEDIR="\n <property>\n <name>general.maven.project.basedir</name>\n <value>${BASE_DIR}</value>\n </property>\n"
+fi
+
+#Configure accumulo-env.sh
+mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/
+sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \
+ -e "s/\${masterHigh_masterLow}/${!MASTER}/" \
+ -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \
+ -e "s/\${gcHigh_gcLow}/${!GC}/" \
+ -e "s/\${otherHigh_otherLow}/${!OTHER}/" \
+ ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV
+
+#Configure accumulo-site.xml
+sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \
+ -e "s/\${nativeEnabled}/${!NATIVE}/" \
+ -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \
+ -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \
+ -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \
+ -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \
+ -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE
+
+#Configure for hadoop 1
+if [[ "$HADOOP_VERSION" = "1" ]]; then
+ sed -e 's/^test -z \"$HADOOP_CONF_DIR\"/#test -z \"$HADOOP_CONF_DIR\"/' -e 's/^# test -z "$HADOOP_CONF_DIR"/test -z \"$HADOOP_CONF_DIR\"/' ${CONF_DIR}/$ACCUMULO_ENV > temp
+ mv temp ${CONF_DIR}/$ACCUMULO_ENV
+ sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements -->\n <!--/' \
+ -e 's/<!-- End Hadoop 2 requirements -->/-->\n <!-- End Hadoop 2 requirements -->/' \
+ ${CONF_DIR}/$ACCUMULO_SITE > temp
+ mv temp ${CONF_DIR}/$ACCUMULO_SITE
+fi
+
+#Additional setup steps for native configuration.
+if [[ "${TYPE}" = "native" ]]; then
+ if [[ "$(uname)" = 'Linux' ]]; then
+ if [[ -z $HADOOP_PREFIX ]]; then
+ echo "HADOOP_PREFIX not set cannot automatically configure LD_LIBRARY_PATH"
+ else
+ NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null)
+ if [[ -z $NATIVE_LIB ]]; then
+ echo -e "Native libraries could not be found for your sytem in: $HADOOP_PREFIX"
+ else
+ sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp
+ mv temp ${CONF_DIR}/$ACCUMULO_ENV
+ echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH"
+ fi
+ fi
+ fi
+ echo -e "Please remember to compile the native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed."
+fi
+echo "Setup complete"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 2a570f4,240de73..9897262
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -16,17 -16,37 +16,26 @@@
*/
package org.apache.accumulo.core;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.ColumnFQ;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
+ import com.google.common.base.Charsets;
+
public class Constants {
- public static final Charset UTF8 = Charset.forName("UTF-8");
+ /**
- * @deprecated since 1.5.3; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
++ * @deprecated since 1.5.3, 1.6.2; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use
+ * {@link Charset#forName(String)} with "UTF-8" instead
+ */
+ @Deprecated
+ public static final Charset UTF8 = UTF_8;
public static final String VERSION = FilteredConstants.VERSION;
-
- // versions should never be negative
- public static final Integer WIRE_VERSION = 2;
- public static final int DATA_VERSION = 5;
- public static final int PREV_DATA_VERSION = 4;
-
+
// Zookeeper locations
public static final String ZROOT = "/accumulo";
public static final String ZINSTANCES = "/instances";
@@@ -56,62 -76,117 +65,62 @@@
public static final String ZMONITOR_LOCK = ZMONITOR + "/lock";
public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr";
public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr";
-
+
public static final String ZCONFIG = "/config";
-
+
public static final String ZTSERVERS = "/tservers";
-
+
public static final String ZDEAD = "/dead";
- public static final String ZDEADTSERVERS = "/dead/tservers";
-
+ public static final String ZDEADTSERVERS = ZDEAD + "/tservers";
+
public static final String ZTRACERS = "/tracers";
-
+
public static final String ZPROBLEMS = "/problems";
- public static final String ZUSERS = "/users";
-
+
public static final String BULK_ARBITRATOR_TYPE = "bulkTx";
-
+
public static final String ZFATE = "/fate";
-
+
public static final String ZNEXT_FILE = "/next_file";
-
+
public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
-
+
public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
public static final String ZRECOVERY = "/recovery";
-
- public static final String METADATA_TABLE_ID = "!0";
- public static final String METADATA_TABLE_NAME = "!METADATA";
- public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
- public static final String TABLE_TABLET_LOCATION = "/table_info";
- public static final String ZTABLE_LOCKS = "/table_locks";
-
- // reserved keyspace is any row that begins with a tilde '~' character
- public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'}));
- public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
- public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
- public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
- public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
- public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
- public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
- public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
- public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
-
- public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv");
- public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet
- public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc");
- public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future");
- public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last");
- public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file
- public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned
-
- // README : very important that prevRow sort last to avoid race conditions between
- // garbage collector and split
- public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything
- // else for that tablet
- public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow"));
- public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir"));
- public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time"));
- public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush"));
- public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact"));
- public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio"));
- public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock"));
-
- public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file");
- public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan");
- public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log");
- public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped");
- public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped"));
-
- public static final Range NON_ROOT_METADATA_KEYSPACE = new Range(
- new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
- public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false);
-
- public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null),
- null);
- public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(
- METADATA_TABLE_ID), null), true);
-
+
/**
- * @deprecated since 1.5.3; not used, but specifies UTF-8
+ * Initial tablet directory name for the default tablet in all tables
*/
- @Deprecated
- public static final String VALUE_ENCODING = UTF_8.name();
-
+ public static final String DEFAULT_TABLET_LOCATION = "/default_tablet";
+
+ public static final String ZTABLE_LOCKS = "/table_locks";
+
public static final String BULK_PREFIX = "b-";
- public static final String OLD_BULK_PREFIX = "bulk_";
-
- // note: all times are in milliseconds
-
- public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata
-
- public static final long MIN_MASTER_LOOP_TIME = 1000;
- public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000;
- public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000;
-
+
+ public static final String CLONE_PREFIX = "c-";
- public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF8);
++ public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF_8);
+
+ // this affects the table client caching of metadata
+ public static final int SCAN_BATCH_SIZE = 1000;
+
+ // Scanners will default to fetching 3 batches of Key/Value pairs before asynchronously
+ // fetching the next batch.
+ public static final long SCANNER_DEFAULT_READAHEAD_THRESHOLD = 3l;
+
// Security configuration
public static final String PW_HASH_ALGORITHM = "SHA-256";
-
- // Representation of an empty set of authorizations
- // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility)
- public static final Authorizations NO_AUTHS = new Authorizations();
-
- public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds
-
+
+ /**
+ * @deprecated since 1.6.0; Use {@link Authorizations#EMPTY} instead
+ */
+ @Deprecated
+ public static final Authorizations NO_AUTHS = Authorizations.EMPTY;
+
public static final int MAX_DATA_TO_PRINT = 64;
- public static final int CLIENT_RETRIES = 5;
- public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core";
- public static final String OLD_PACKAGE_NAME = "cloudbase";
- public static final String VALID_TABLE_NAME_REGEX = "^\\w+$";
public static final String MAPFILE_EXTENSION = "map";
public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-";
-
+
public static final String EXPORT_METADATA_FILE = "metadata.bin";
public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt";
public static final String EXPORT_FILE = "exportMetadata.zip";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
index 35a9067,5134521..041c9e6
--- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java
@@@ -16,6 -16,9 +16,8 @@@
*/
package org.apache.accumulo.core.cli;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 6d35757,251d6a7..a9c3292
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@@ -16,6 -16,11 +16,8 @@@
*/
package org.apache.accumulo.core.client;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
index 63e3497,0000000..6925f18
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java
@@@ -1,210 -1,0 +1,212 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.client.admin.ActiveCompaction;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ConfigurationType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Provides a class for administering the accumulo instance
+ */
+public class InstanceOperationsImpl implements InstanceOperations {
+ private Instance instance;
+ private Credentials credentials;
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the Credential, containing principal and Authentication Token
+ */
+ public InstanceOperationsImpl(Instance instance, Credentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ @Override
+ public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property, value);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property, value);
+ }
+ });
+ }
+
+ @Override
+ public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(property);
+ MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property);
+ }
+ });
+ }
+
+ @Override
+ public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.CURRENT);
+ }
+ });
+ }
+
+ @Override
+ public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.SITE);
+ }
+ });
+ }
+
+ @Override
+ public List<String> getTabletServers() {
+ ZooCache cache = new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ List<String> results = new ArrayList<String>();
+ for (String candidate : cache.getChildren(path)) {
+ List<String> children = cache.getChildren(path + "/" + candidate);
+ if (children != null && children.size() > 0) {
+ List<String> copy = new ArrayList<String>(children);
+ Collections.sort(copy);
+ byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0));
- if (data != null && !"master".equals(new String(data, Constants.UTF8))) {
++ if (data != null && !"master".equals(new String(data, UTF_8))) {
+ results.add(candidate);
+ }
+ }
+ }
+ return results;
+ }
+
+ @Override
+ public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
+
+ List<ActiveScan> as = new ArrayList<ActiveScan>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials.toThrift(instance))) {
+ try {
+ as.add(new ActiveScanImpl(instance, activeScan));
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException {
+ return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkClass(Tracer.traceInfo(), credentials.toThrift(instance), className, asTypeName);
+ }
+ });
+ }
+
+ @Override
+ public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
+ Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance));
+
+ List<ActiveCompaction> as = new ArrayList<ActiveCompaction>();
+ for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(),
+ credentials.toThrift(instance))) {
+ as.add(new ActiveCompactionImpl(instance, activeCompaction));
+ }
+ return as;
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ ThriftUtil.returnClient(client);
+ }
+ }
+
+ @Override
+ public void ping(String tserver) throws AccumuloException {
+ TTransport transport = null;
+ try {
+ transport = ThriftUtil.createTransport(AddressUtil.parseAddress(tserver, false), ServerConfigurationUtil.getConfiguration(instance));
+ TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport);
+ client.getTabletServerStatus(Tracer.traceInfo(), credentials.toThrift(instance));
+ } catch (TTransportException e) {
+ throw new AccumuloException(e);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloException(e);
+ } catch (TException e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (transport != null) {
+ transport.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
index 30f0266,0000000..f32940c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java
@@@ -1,154 -1,0 +1,156 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.security.SecurityPermission;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.util.ArgumentChecker.Validator;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+
+public class Namespaces {
+ public static final String VALID_NAME_REGEX = "^\\w*$";
+ public static final Validator<String> VALID_NAME = new Validator<String>() {
+ @Override
+ public boolean isValid(String namespace) {
+ return namespace != null && namespace.matches(VALID_NAME_REGEX);
+ }
+
+ @Override
+ public String invalidMessage(String namespace) {
+ if (namespace == null)
+ return "Namespace cannot be null";
+ return "Namespaces must only contain word characters (letters, digits, and underscores): " + namespace;
+ }
+ };
+
+ public static final Validator<String> NOT_DEFAULT = new Validator<String>() {
+ @Override
+ public boolean isValid(String namespace) {
+ return !Namespaces.DEFAULT_NAMESPACE.equals(namespace);
+ }
+
+ @Override
+ public String invalidMessage(String namespace) {
+ return "Namespace cannot be the reserved empty namespace";
+ }
+ };
+
+ public static final Validator<String> NOT_ACCUMULO = new Validator<String>() {
+ @Override
+ public boolean isValid(String namespace) {
+ return !Namespaces.ACCUMULO_NAMESPACE.equals(namespace);
+ }
+
+ @Override
+ public String invalidMessage(String namespace) {
+ return "Namespace cannot be the reserved namespace, " + Namespaces.ACCUMULO_NAMESPACE;
+ }
+ };
+
+ private static SecurityPermission TABLES_PERMISSION = new SecurityPermission("tablesPermission");
+
+ public static final String DEFAULT_NAMESPACE_ID = "+default";
+ public static final String DEFAULT_NAMESPACE = "";
+ public static final String ACCUMULO_NAMESPACE_ID = "+accumulo";
+ public static final String ACCUMULO_NAMESPACE = "accumulo";
+
+ private static ZooCache getZooCache(Instance instance) {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TABLES_PERMISSION);
+ }
+ return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ }
+
+ private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
+ ZooCache zc = getZooCache(instance);
+
+ List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
+
+ TreeMap<String,String> namespaceMap = new TreeMap<String,String>();
+
+ for (String id : namespaceIds) {
+ byte[] path = zc.get(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + id + Constants.ZNAMESPACE_NAME);
+ if (path != null) {
+ if (nameAsKey)
- namespaceMap.put(new String(path, Constants.UTF8), id);
++ namespaceMap.put(new String(path, UTF_8), id);
+ else
- namespaceMap.put(id, new String(path, Constants.UTF8));
++ namespaceMap.put(id, new String(path, UTF_8));
+ }
+ }
+ return namespaceMap;
+ }
+
+ public static boolean exists(Instance instance, String namespaceId) {
+ ZooCache zc = getZooCache(instance);
+ List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES);
+ return namespaceIds.contains(namespaceId);
+ }
+
+ public static String getNamespaceId(Instance instance, String namespace) throws NamespaceNotFoundException {
+ String id = getNameToIdMap(instance).get(namespace);
+ if (id == null)
+ throw new NamespaceNotFoundException(null, namespace, "getNamespaceId() failed to find namespace");
+ return id;
+ }
+
+ public static String getNamespaceName(Instance instance, String namespaceId) throws NamespaceNotFoundException {
+ String namespaceName = getIdToNameMap(instance).get(namespaceId);
+ if (namespaceName == null)
+ throw new NamespaceNotFoundException(namespaceId, null, "getNamespaceName() failed to find namespace");
+ return namespaceName;
+ }
+
+ public static SortedMap<String,String> getNameToIdMap(Instance instance) {
+ return getMap(instance, true);
+ }
+
+ public static SortedMap<String,String> getIdToNameMap(Instance instance) {
+ return getMap(instance, false);
+ }
+
+ public static List<String> getTableIds(Instance instance, String namespaceId) throws NamespaceNotFoundException {
+ String namespace = getNamespaceName(instance, namespaceId);
+ List<String> names = new LinkedList<String>();
+ for (Entry<String,String> nameToId : Tables.getNameToIdMap(instance).entrySet())
+ if (namespace.equals(Tables.qualify(nameToId.getKey()).getFirst()))
+ names.add(nameToId.getValue());
+ return names;
+ }
+
+ public static List<String> getTableNames(Instance instance, String namespaceId) throws NamespaceNotFoundException {
+ String namespace = getNamespaceName(instance, namespaceId);
+ List<String> names = new LinkedList<String>();
+ for (String name : Tables.getNameToIdMap(instance).keySet())
+ if (namespace.equals(Tables.qualify(name).getFirst()))
+ names.add(name);
+ return names;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
index 8da530b,1fcebe7..70bdbb0
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java
@@@ -16,7 -16,11 +16,9 @@@
*/
package org.apache.accumulo.core.client.impl;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
@@@ -128,10 -142,13 +130,10 @@@ public class ServerClient
for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
byte[] data = ZooUtil.getLockData(zc, path);
- if (data != null && !new String(data, Constants.UTF8).equals("master"))
- if (data != null) {
- String tserverData = new String(data, UTF_8);
- if (!tserverData.equals("master")) {
- servers.add(new ThriftTransportKey(new ServerServices(tserverData).getAddressString(Service.TSERV_CLIENT), conf.getPort(Property.TSERV_CLIENTPORT),
- rpcTimeout));
- }
- }
++ if (data != null && !new String(data, UTF_8).equals("master"))
+ servers.add(new ThriftTransportKey(
+ new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT),
+ rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance))));
}
boolean opened = false;
[29/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index b1bbb67,0000000..4f381ef
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@@ -1,112 -1,0 +1,114 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+
+public class RenameTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+ private String oldTableName;
+ private String newTableName;
+ private String namespaceId;
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.RENAME) + Utils.reserveTable(tableId, tid, true, true, TableOperation.RENAME);
+ }
+
+ public RenameTable(String tableId, String oldTableName, String newTableName) throws NamespaceNotFoundException {
+ this.tableId = tableId;
+ this.oldTableName = oldTableName;
+ this.newTableName = newTableName;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+
+ Instance instance = master.getInstance();
+ Pair<String,String> qualifiedOldTableName = Tables.qualify(oldTableName);
+ Pair<String,String> qualifiedNewTableName = Tables.qualify(newTableName);
+
+ // ensure no attempt is made to rename across namespaces
+ if (newTableName.contains(".") && !namespaceId.equals(Namespaces.getNamespaceId(instance, qualifiedNewTableName.getFirst())))
+ throw new ThriftTableOperationException(tableId, oldTableName, TableOperation.RENAME, TableOperationExceptionType.INVALID_NAME,
+ "Namespace in new table name does not match the old table name");
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ Utils.tableNameLock.lock();
+ try {
+ Utils.checkTableDoesNotExist(instance, newTableName, tableId, TableOperation.RENAME);
+
+ final String newName = qualifiedNewTableName.getSecond();
+ final String oldName = qualifiedOldTableName.getSecond();
+
+ final String tap = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME;
+
+ zoo.mutate(tap, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] current) throws Exception {
- final String currentName = new String(current, Constants.UTF8);
++ final String currentName = new String(current, UTF_8);
+ if (currentName.equals(newName))
+ return null; // assume in this case the operation is running again, so we are done
+ if (!currentName.equals(oldName)) {
+ throw new ThriftTableOperationException(null, oldTableName, TableOperation.RENAME, TableOperationExceptionType.NOTFOUND,
+ "Name changed while processing");
+ }
- return newName.getBytes(Constants.UTF8);
++ return newName.getBytes(UTF_8);
+ }
+ });
+ Tables.clearCache(instance);
+ } finally {
+ Utils.tableNameLock.unlock();
+ Utils.unreserveTable(tableId, tid, true);
+ Utils.unreserveNamespace(this.namespaceId, tid, false);
+ }
+
+ Logger.getLogger(RenameTable.class).debug("Renamed table " + tableId + " " + oldTableName + " " + newTableName);
+
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveTable(tableId, tid, true);
+ Utils.unreserveNamespace(namespaceId, tid, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 7ede305,0000000..3e8704e
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -1,163 -1,0 +1,165 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.math.BigInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.fate.zookeeper.ZooReservation;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooQueueLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class Utils {
+ private static final byte[] ZERO_BYTE = new byte[] {'0'};
+
+ static void checkTableDoesNotExist(Instance instance, String tableName, String tableId, TableOperation operation) throws ThriftTableOperationException {
+
+ String id = Tables.getNameToIdMap(instance).get(tableName);
+
+ if (id != null && !id.equals(tableId))
+ throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.EXISTS, null);
+ }
+
+ static String getNextTableId(String tableName, Instance instance) throws ThriftTableOperationException {
+
+ String tableId = null;
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ final String ntp = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ byte[] nid = zoo.mutate(ntp, ZERO_BYTE, ZooUtil.PUBLIC, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- BigInteger nextId = new BigInteger(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++ BigInteger nextId = new BigInteger(new String(currentValue, UTF_8), Character.MAX_RADIX);
+ nextId = nextId.add(BigInteger.ONE);
- return nextId.toString(Character.MAX_RADIX).getBytes(Constants.UTF8);
++ return nextId.toString(Character.MAX_RADIX).getBytes(UTF_8);
+ }
+ });
- return new String(nid, Constants.UTF8);
++ return new String(nid, UTF_8);
+ } catch (Exception e1) {
+ Logger.getLogger(CreateTable.class).error("Failed to assign tableId to " + tableName, e1);
+ throw new ThriftTableOperationException(tableId, tableName, TableOperation.CREATE, TableOperationExceptionType.OTHER, e1.getMessage());
+ }
+ }
+
+ static final Lock tableNameLock = new ReentrantLock();
+ static final Lock idLock = new ReentrantLock();
+ private static final Logger log = Logger.getLogger(Utils.class);
+
+ public static long reserveTable(String tableId, long tid, boolean writeLock, boolean tableMustExist, TableOperation op) throws Exception {
+ if (getLock(tableId, tid, writeLock).tryLock()) {
+ if (tableMustExist) {
+ Instance instance = HdfsZooInstance.getInstance();
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId))
+ throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist");
+ }
+ log.info("table " + tableId + " (" + Long.toHexString(tid) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
+ return 0;
+ } else
+ return 100;
+ }
+
+ public static void unreserveTable(String tableId, long tid, boolean writeLock) throws Exception {
+ getLock(tableId, tid, writeLock).unlock();
+ log.info("table " + tableId + " (" + Long.toHexString(tid) + ") unlocked for " + (writeLock ? "write" : "read"));
+ }
+
+ public static void unreserveNamespace(String namespaceId, long id, boolean writeLock) throws Exception {
+ getLock(namespaceId, id, writeLock).unlock();
+ log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") unlocked for " + (writeLock ? "write" : "read"));
+ }
+
+ public static long reserveNamespace(String namespaceId, long id, boolean writeLock, boolean mustExist, TableOperation op) throws Exception {
+ if (getLock(namespaceId, id, writeLock).tryLock()) {
+ if (mustExist) {
+ Instance instance = HdfsZooInstance.getInstance();
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId))
+ throw new ThriftTableOperationException(namespaceId, "", op, TableOperationExceptionType.NAMESPACE_NOTFOUND, "Namespace does not exist");
+ }
+ log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
+ return 0;
+ } else
+ return 100;
+ }
+
+ public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
+ Instance instance = HdfsZooInstance.getInstance();
+
+ String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
- + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++ + Base64.encodeBase64String(directory.getBytes(UTF_8));
+
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+
+ if (ZooReservation.attempt(zk, resvPath, String.format("%016x", tid), "")) {
+ return 0;
+ } else
+ return 50;
+ }
+
+ public static void unreserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
+ Instance instance = HdfsZooInstance.getInstance();
+ String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
- + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++ + Base64.encodeBase64String(directory.getBytes(UTF_8));
+ ZooReservation.release(ZooReaderWriter.getInstance(), resvPath, String.format("%016x", tid));
+ }
+
+ private static Lock getLock(String tableId, long tid, boolean writeLock) throws Exception {
- byte[] lockData = String.format("%016x", tid).getBytes(Constants.UTF8);
++ byte[] lockData = String.format("%016x", tid).getBytes(UTF_8);
+ ZooQueueLock qlock = new ZooQueueLock(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLE_LOCKS + "/" + tableId, false);
+ Lock lock = DistributedReadWriteLock.recoverLock(qlock, lockData);
+ if (lock == null) {
+ DistributedReadWriteLock locker = new DistributedReadWriteLock(qlock, lockData);
+ if (writeLock)
+ lock = locker.writeLock();
+ else
+ lock = locker.readLock();
+ }
+ return lock;
+ }
+
+ public static Lock getReadLock(String tableId, long tid) throws Exception {
+ return Utils.getLock(tableId, tid, false);
+ }
+
+ static void checkNamespaceDoesNotExist(Instance instance, String namespace, String namespaceId, TableOperation operation)
+ throws ThriftTableOperationException {
+
+ String n = Namespaces.getNameToIdMap(instance).get(namespace);
+
+ if (n != null && !n.equals(namespaceId))
+ throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.NAMESPACE_EXISTS, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
index 20b7328,0000000..a9058b7
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
@@@ -1,92 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tserverOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.master.EventCoordinator.Listener;
+import org.apache.accumulo.master.tableOps.MasterRepo;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransportException;
+
+public class ShutdownTServer extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger log = Logger.getLogger(ShutdownTServer.class);
+ private TServerInstance server;
+ private boolean force;
+
+ public ShutdownTServer(TServerInstance server, boolean force) {
+ this.server = server;
+ this.force = force;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ // suppress assignment of tablets to the server
+ if (force) {
+ String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZTSERVERS + "/" + server.getLocation();
+ ZooLock.deleteLock(path);
+ path = ZooUtil.getRoot(master.getInstance()) + Constants.ZDEADTSERVERS + "/" + server.getLocation();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- zoo.putPersistentData(path, "forced down".getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ zoo.putPersistentData(path, "forced down".getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ return null;
+ }
+
+ // TODO move this to isReady() and drop while loop? - ACCUMULO-1259
+ Listener listener = master.getEventCoordinator().getListener();
+ master.shutdownTServer(server);
+ while (master.onlineTabletServers().contains(server)) {
+ TServerConnection connection = master.getConnection(server);
+ if (connection != null) {
+ try {
+ TabletServerStatus status = connection.getTableMap(false);
+ if (status.tableMap != null && status.tableMap.isEmpty()) {
+ log.info("tablet server hosts no tablets " + server);
+ connection.halt(master.getMasterLock());
+ log.info("tablet server asked to halt " + server);
+ break;
+ }
+ } catch (TTransportException ex) {
+ // expected
+ } catch (Exception ex) {
+ log.error("Error talking to tablet server " + server + ": " + ex);
+ }
+ }
+ listener.waitForEvents(1000);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master m) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index c4ab6e6,0000000..75d1e9e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -1,817 -1,0 +1,819 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService;
+import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.monitor.servlets.DefaultServlet;
+import org.apache.accumulo.monitor.servlets.GcStatusServlet;
+import org.apache.accumulo.monitor.servlets.JSONServlet;
+import org.apache.accumulo.monitor.servlets.LogServlet;
+import org.apache.accumulo.monitor.servlets.MasterServlet;
+import org.apache.accumulo.monitor.servlets.OperationServlet;
+import org.apache.accumulo.monitor.servlets.ProblemServlet;
+import org.apache.accumulo.monitor.servlets.ScanServlet;
+import org.apache.accumulo.monitor.servlets.ShellServlet;
+import org.apache.accumulo.monitor.servlets.TServersServlet;
+import org.apache.accumulo.monitor.servlets.TablesServlet;
+import org.apache.accumulo.monitor.servlets.VisServlet;
+import org.apache.accumulo.monitor.servlets.XMLServlet;
+import org.apache.accumulo.monitor.servlets.trace.ListType;
+import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
+import org.apache.accumulo.monitor.servlets.trace.Summary;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.monitor.LogService;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.TableInfoUtil;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * Serve master statistics with an embedded web server.
+ */
+public class Monitor {
+ private static final Logger log = Logger.getLogger(Monitor.class);
+
+ private static final int REFRESH_TIME = 5;
+ private static long lastRecalc = 0L;
+ private static double totalIngestRate = 0.0;
+ private static double totalIngestByteRate = 0.0;
+ private static double totalQueryRate = 0.0;
+ private static double totalScanRate = 0.0;
+ private static double totalQueryByteRate = 0.0;
+ private static long totalEntries = 0L;
+ private static int totalTabletCount = 0;
+ private static int onlineTabletCount = 0;
+ private static long totalHoldTime = 0;
+ private static long totalLookups = 0;
+ private static int totalTables = 0;
+
+ private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
+ private static final long serialVersionUID = 1L;
+
+ private long maxDelta;
+
+ public MaxList(long maxDelta) {
+ this.maxDelta = maxDelta;
+ }
+
+ @Override
+ public boolean add(Pair<Long,T> obj) {
+ boolean result = super.add(obj);
+
+ if (obj.getFirst() - get(0).getFirst() > maxDelta)
+ remove(0);
+
+ return result;
+ }
+
+ }
+
+ private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
+ private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static EventCounter lookupRateTracker = new EventCounter();
+ private static EventCounter indexCacheHitTracker = new EventCounter();
+ private static EventCounter indexCacheRequestTracker = new EventCounter();
+ private static EventCounter dataCacheHitTracker = new EventCounter();
+ private static EventCounter dataCacheRequestTracker = new EventCounter();
+
+ private static volatile boolean fetching = false;
+ private static MasterMonitorInfo mmi;
+ private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
+ private static Exception problemException;
+ private static GCStatus gcStatus;
+
+ private static Instance instance;
+
+ private static ServerConfiguration config;
+
+ private static EmbeddedWebServer server;
+
+ private ZooLock monitorLock;
+
+ private static class EventCounter {
+
+ Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
+ Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
+ Set<String> serversUpdated = new HashSet<String>();
+
+ void startingUpdates() {
+ serversUpdated.clear();
+ }
+
+ void updateTabletServer(String name, long sampleTime, long numEvents) {
+ Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
+ Pair<Long,Long> lastSample = samples.get(name);
+
+ if (lastSample == null || !lastSample.equals(newSample)) {
+ samples.put(name, newSample);
+ if (lastSample != null) {
+ prevSamples.put(name, lastSample);
+ }
+ }
+ serversUpdated.add(name);
+ }
+
+ void finishedUpdating() {
+ // remove any tablet servers not updated
+ samples.keySet().retainAll(serversUpdated);
+ prevSamples.keySet().retainAll(serversUpdated);
+ }
+
+ double calculateRate() {
+ double totalRate = 0;
+
+ for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+ Pair<Long,Long> prevSample = entry.getValue();
+ Pair<Long,Long> sample = samples.get(entry.getKey());
+
+ totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
+ }
+
+ return totalRate;
+ }
+
+ long calculateCount() {
+ long count = 0;
+
+ for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+ Pair<Long,Long> prevSample = entry.getValue();
+ Pair<Long,Long> sample = samples.get(entry.getKey());
+
+ count += sample.getSecond() - prevSample.getSecond();
+ }
+
+ return count;
+ }
+ }
+
+ public static void fetchData() {
+ double totalIngestRate = 0.;
+ double totalIngestByteRate = 0.;
+ double totalQueryRate = 0.;
+ double totalQueryByteRate = 0.;
+ double totalScanRate = 0.;
+ long totalEntries = 0;
+ int totalTabletCount = 0;
+ int onlineTabletCount = 0;
+ long totalHoldTime = 0;
+ long totalLookups = 0;
+ boolean retry = true;
+
+ // only recalc every so often
+ long currentTime = System.currentTimeMillis();
+ if (currentTime - lastRecalc < REFRESH_TIME * 1000)
+ return;
+
+ synchronized (Monitor.class) {
+ if (fetching)
+ return;
+ fetching = true;
+ }
+
+ try {
+ while (retry) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnection(HdfsZooInstance.getInstance());
+ if (client != null) {
+ mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
+ retry = false;
+ } else {
+ mmi = null;
+ }
+ Monitor.gcStatus = fetchGcStatus();
+ } catch (Exception e) {
+ mmi = null;
+ log.info("Error fetching stats: " + e);
+ } finally {
+ if (client != null) {
+ MasterClient.close(client);
+ }
+ }
+ if (mmi == null)
+ UtilWaitThread.sleep(1000);
+ }
+ if (mmi != null) {
+ int majorCompactions = 0;
+ int minorCompactions = 0;
+
+ lookupRateTracker.startingUpdates();
+ indexCacheHitTracker.startingUpdates();
+ indexCacheRequestTracker.startingUpdates();
+ dataCacheHitTracker.startingUpdates();
+ dataCacheRequestTracker.startingUpdates();
+
+ for (TabletServerStatus server : mmi.tServerInfo) {
+ TableInfo summary = TableInfoUtil.summarizeTableStats(server);
+ totalIngestRate += summary.ingestRate;
+ totalIngestByteRate += summary.ingestByteRate;
+ totalQueryRate += summary.queryRate;
+ totalScanRate += summary.scanRate;
+ totalQueryByteRate += summary.queryByteRate;
+ totalEntries += summary.recs;
+ totalHoldTime += server.holdTime;
+ totalLookups += server.lookups;
+ majorCompactions += summary.majors.running;
+ minorCompactions += summary.minors.running;
+ lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
+ indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
+ indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
+ dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
+ dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
+ }
+
+ lookupRateTracker.finishedUpdating();
+ indexCacheHitTracker.finishedUpdating();
+ indexCacheRequestTracker.finishedUpdating();
+ dataCacheHitTracker.finishedUpdating();
+ dataCacheRequestTracker.finishedUpdating();
+
+ int totalTables = 0;
+ for (TableInfo tInfo : mmi.tableMap.values()) {
+ totalTabletCount += tInfo.tablets;
+ onlineTabletCount += tInfo.onlineTablets;
+ totalTables++;
+ }
+ Monitor.totalIngestRate = totalIngestRate;
+ Monitor.totalTables = totalTables;
+ totalIngestByteRate = totalIngestByteRate / 1000000.0;
+ Monitor.totalIngestByteRate = totalIngestByteRate;
+ Monitor.totalQueryRate = totalQueryRate;
+ Monitor.totalScanRate = totalScanRate;
+ totalQueryByteRate = totalQueryByteRate / 1000000.0;
+ Monitor.totalQueryByteRate = totalQueryByteRate;
+ Monitor.totalEntries = totalEntries;
+ Monitor.totalTabletCount = totalTabletCount;
+ Monitor.onlineTabletCount = onlineTabletCount;
+ Monitor.totalHoldTime = totalHoldTime;
+ Monitor.totalLookups = totalLookups;
+
+ ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
+ ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
+
+ double totalLoad = 0.;
+ for (TabletServerStatus status : mmi.tServerInfo) {
+ if (status != null)
+ totalLoad += status.osLoad;
+ }
+ loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
+
+ minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
+ majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
+
+ lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
+
+ queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
+ queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
+
+ scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
+
+ calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
+ calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
+ }
+ try {
+ Monitor.problemSummary = ProblemReports.getInstance().summarize();
+ Monitor.problemException = null;
+ } catch (Exception e) {
+ log.info("Failed to obtain problem reports ", e);
+ Monitor.problemSummary = Collections.emptyMap();
+ Monitor.problemException = e;
+ }
+
+ } finally {
+ synchronized (Monitor.class) {
+ fetching = false;
+ lastRecalc = currentTime;
+ }
+ }
+ }
+
+ private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
+ long req = cacheReq.calculateCount();
+ if (req > 0)
+ hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
+ else
+ hitRate.add(new Pair<Long,Double>(currentTime, null));
+ }
+
+ private static GCStatus fetchGcStatus() {
+ GCStatus result = null;
+ HostAndPort address = null;
+ try {
+ // Read the gc location from its lock
+ ZooReaderWriter zk = ZooReaderWriter.getInstance();
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ List<String> locks = zk.getChildren(path, null);
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
- address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), Constants.UTF8)).getAddress(Service.GC_CLIENT);
++ address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), UTF_8)).getAddress(Service.GC_CLIENT);
+ GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
+ try {
+ result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+ } catch (Exception ex) {
+ log.warn("Unable to contact the garbage collector at " + address, ex);
+ }
+ return result;
+ }
+
+ public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+
+ ServerOpts opts = new ServerOpts();
+ final String app = "monitor";
+ opts.parseArgs(app, args);
+ String hostname = opts.getAddress();
+
+ Accumulo.setupLogging(app);
+ VolumeManager fs = VolumeManagerImpl.get();
+ instance = HdfsZooInstance.getInstance();
+ config = new ServerConfiguration(instance);
+ Accumulo.init(fs, config, app);
+ Monitor monitor = new Monitor();
+ Accumulo.enableTracing(hostname, app);
+ monitor.run(hostname);
+ }
+
+ private static long START_TIME;
+
+ public void run(String hostname) {
+ try {
+ getMonitorLock();
+ } catch (Exception e) {
+ log.error("Failed to get Monitor ZooKeeper lock");
+ throw new RuntimeException(e);
+ }
+
+ Monitor.START_TIME = System.currentTimeMillis();
+ int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
+ try {
+ log.debug("Creating monitor on port " + port);
+ server = new EmbeddedWebServer(hostname, port);
+ } catch (Throwable ex) {
+ log.error("Unable to start embedded web server", ex);
+ throw new RuntimeException(ex);
+ }
+
+ server.addServlet(DefaultServlet.class, "/");
+ server.addServlet(OperationServlet.class, "/op");
+ server.addServlet(MasterServlet.class, "/master");
+ server.addServlet(TablesServlet.class, "/tables");
+ server.addServlet(TServersServlet.class, "/tservers");
+ server.addServlet(ProblemServlet.class, "/problems");
+ server.addServlet(GcStatusServlet.class, "/gc");
+ server.addServlet(LogServlet.class, "/log");
+ server.addServlet(XMLServlet.class, "/xml");
+ server.addServlet(JSONServlet.class, "/json");
+ server.addServlet(VisServlet.class, "/vis");
+ server.addServlet(ScanServlet.class, "/scans");
+ server.addServlet(Summary.class, "/trace/summary");
+ server.addServlet(ListType.class, "/trace/listType");
+ server.addServlet(ShowTrace.class, "/trace/show");
+ if (server.isUsingSsl())
+ server.addServlet(ShellServlet.class, "/shell");
+ server.start();
+
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+
+ log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
+
+ String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
+
- ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(Constants.UTF8),
++ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(UTF_8),
+ NodeExistsPolicy.OVERWRITE);
+ log.info("Set monitor address in zookeeper to " + monitorAddress);
+ } catch (Exception ex) {
+ log.error("Unable to set monitor HTTP address in zookeeper", ex);
+ }
+
+ if (null != hostname) {
+ LogService.startLogListener(Monitor.getSystemConfiguration(), instance.getInstanceID(), hostname);
+ } else {
+ log.warn("Not starting log4j listener as we could not determine address to use");
+ }
+
+ new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
+
+ // need to regularly fetch data so plot data is updated
+ new Daemon(new LoggingRunnable(log, new Runnable() {
+
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Monitor.fetchData();
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+
+ UtilWaitThread.sleep(333);
+ }
+
+ }
+ }), "Data fetcher").start();
+
+ new Daemon(new LoggingRunnable(log, new Runnable() {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Monitor.fetchScans();
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ UtilWaitThread.sleep(5000);
+ }
+ }
+ }), "Scan scanner").start();
+ }
+
+ public static class ScanStats {
+ public final List<ActiveScan> scans;
+ public final long fetched;
+ ScanStats(List<ActiveScan> active) {
+ this.scans = active;
+ this.fetched = System.currentTimeMillis();
+ }
+ }
+ static final Map<String, ScanStats> allScans = new HashMap<String, ScanStats>();
+ public static Map<String, ScanStats> getScans() {
+ synchronized (allScans) {
+ return new TreeMap<String, ScanStats>(allScans);
+ }
+ }
+
+ protected static void fetchScans() throws Exception {
+ if (instance == null)
+ return;
+ Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ for (String server : c.instanceOperations().getTabletServers()) {
+ Client tserver = ThriftUtil.getTServerClient(server, Monitor.getSystemConfiguration());
+ try {
+ List<ActiveScan> scans = tserver.getActiveScans(null, SystemCredentials.get().toThrift(instance));
+ synchronized (allScans) {
+ allScans.put(server, new ScanStats(scans));
+ }
+ } catch (Exception ex) {
+ log.debug(ex, ex);
+ } finally {
+ ThriftUtil.returnClient(tserver);
+ }
+ }
+ // Age off old scan information
+ Iterator<Entry<String,ScanStats>> entryIter = allScans.entrySet().iterator();
+ long now = System.currentTimeMillis();
+ while (entryIter.hasNext()) {
+ Entry<String,ScanStats> entry = entryIter.next();
+ if (now - entry.getValue().fetched > 5 * 60 * 1000) {
+ entryIter.remove();
+ }
+ }
+ }
+
+ /**
+ * Get the monitor lock in ZooKeeper
+ */
+ private void getMonitorLock() throws KeeperException, InterruptedException {
+ final String zRoot = ZooUtil.getRoot(instance);
+ final String monitorPath = zRoot + Constants.ZMONITOR;
+ final String monitorLockPath = zRoot + Constants.ZMONITOR_LOCK;
+
+ // Ensure that everything is kosher with ZK as this has changed.
+ ZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.exists(monitorPath)) {
+ byte[] data = zoo.getData(monitorPath, null);
+ // If the node isn't empty, it's from a previous install (has hostname:port for HTTP server)
+ if (0 != data.length) {
+ // Recursively delete from that parent node
+ zoo.recursiveDelete(monitorPath, NodeMissingPolicy.SKIP);
+
+ // And then make the nodes that we expect for the incoming ephemeral nodes
+ zoo.putPersistentData(monitorPath, new byte[0], NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
+ } else if (!zoo.exists(monitorLockPath)) {
+ // monitor node in ZK exists and is empty as we expect
+ // but the monitor/lock node does not
+ zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
+ }
+ } else {
+ // 1.5.0 and earlier
+ zoo.putPersistentData(zRoot + Constants.ZMONITOR, new byte[0], NodeExistsPolicy.FAIL);
+ if (!zoo.exists(monitorLockPath)) {
+ // Somehow the monitor node exists but not monitor/lock
+ zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
+ }
+ }
+
+ // Get a ZooLock for the monitor
+ while (true) {
+ MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher();
+ monitorLock = new ZooLock(monitorLockPath);
+ monitorLock.lockAsync(monitorLockWatcher, new byte[0]);
+
+ monitorLockWatcher.waitForChange();
+
+ if (monitorLockWatcher.acquiredLock) {
+ break;
+ }
+
+ if (!monitorLockWatcher.failedToAcquireLock) {
+ throw new IllegalStateException("monitor lock in unknown state");
+ }
+
+ monitorLock.tryToCancelAsyncLockOrUnlock();
+
+ UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));
+ }
+
+ log.info("Got Monitor lock.");
+ }
+
+ /**
+ * Async Watcher for monitor lock
+ */
+ private static class MoniterLockWatcher implements ZooLock.AsyncLockWatcher {
+
+ boolean acquiredLock = false;
+ boolean failedToAcquireLock = false;
+
+ @Override
+ public void lostLock(LockLossReason reason) {
+ Halt.halt("Monitor lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
+ }
+
+ @Override
+ public void unableToMonitorLockNode(final Throwable e) {
+ Halt.halt(-1, new Runnable() {
+ @Override
+ public void run() {
+ log.fatal("No longer able to monitor Monitor lock node", e);
+ }
+ });
+
+ }
+
+ @Override
+ public synchronized void acquiredLock() {
+ if (acquiredLock || failedToAcquireLock) {
+ Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ acquiredLock = true;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void failedToAcquireLock(Exception e) {
+ log.warn("Failed to get monitor lock " + e);
+
+ if (acquiredLock) {
+ Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ failedToAcquireLock = true;
+ notifyAll();
+ }
+
+ public synchronized void waitForChange() {
+ while (!acquiredLock && !failedToAcquireLock) {
+ try {
+ wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+
+ public static MasterMonitorInfo getMmi() {
+ return mmi;
+ }
+
+ public static int getTotalTables() {
+ return totalTables;
+ }
+
+ public static int getTotalTabletCount() {
+ return totalTabletCount;
+ }
+
+ public static int getOnlineTabletCount() {
+ return onlineTabletCount;
+ }
+
+ public static long getTotalEntries() {
+ return totalEntries;
+ }
+
+ public static double getTotalIngestRate() {
+ return totalIngestRate;
+ }
+
+ public static double getTotalIngestByteRate() {
+ return totalIngestByteRate;
+ }
+
+ public static double getTotalQueryRate() {
+ return totalQueryRate;
+ }
+
+ public static double getTotalScanRate() {
+ return totalScanRate;
+ }
+
+ public static double getTotalQueryByteRate() {
+ return totalQueryByteRate;
+ }
+
+ public static long getTotalHoldTime() {
+ return totalHoldTime;
+ }
+
+ public static Exception getProblemException() {
+ return problemException;
+ }
+
+ public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
+ return problemSummary;
+ }
+
+ public static GCStatus getGcStatus() {
+ return gcStatus;
+ }
+
+ public static long getTotalLookups() {
+ return totalLookups;
+ }
+
+ public static long getStartTime() {
+ return START_TIME;
+ }
+
+ public static List<Pair<Long,Double>> getLoadOverTime() {
+ synchronized (loadOverTime) {
+ return new ArrayList<Pair<Long,Double>>(loadOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIngestRateOverTime() {
+ synchronized (ingestRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
+ synchronized (ingestByteRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
+ synchronized (recoveriesOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
+ synchronized (minorCompactionsOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
+ synchronized (majorCompactionsOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getLookupsOverTime() {
+ synchronized (lookupsOverTime) {
+ return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
+ }
+ }
+
+ public static double getLookupRate() {
+ return lookupRateTracker.calculateRate();
+ }
+
+ public static List<Pair<Long,Integer>> getQueryRateOverTime() {
+ synchronized (queryRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getScanRateOverTime() {
+ synchronized (scanRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
+ synchronized (queryByteRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
+ synchronized (indexCacheHitRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
+ synchronized (dataCacheHitRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
+ }
+ }
+
+ public static AccumuloConfiguration getSystemConfiguration() {
+ return config.getConfiguration();
+ }
+
+ public static Instance getInstance() {
+ return instance;
+ }
+
+ public static boolean isUsingSsl() {
+ return server.isUsingSsl();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index 5e3e475,0000000..78dd95e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -1,140 -1,0 +1,141 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collection;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.TTimeoutTransport;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+import com.google.common.net.HostAndPort;
+
+public class ZooKeeperStatus implements Runnable {
+
+ private static final Logger log = Logger.getLogger(ZooKeeperStatus.class);
+
+ private volatile boolean stop = false;
+
+ public static class ZooKeeperState implements Comparable<ZooKeeperState> {
+ public final String keeper;
+ public final String mode;
+ public final int clients;
+
+ public ZooKeeperState(String keeper, String mode, int clients) {
+ this.keeper = keeper;
+ this.mode = mode;
+ this.clients = clients;
+ }
+
+ @Override
+ public int compareTo(ZooKeeperState other) {
+ if (this == other) {
+ return 0;
+ } else if (other == null) {
+ return 1;
+ } else {
+ if (this.keeper == other.keeper) {
+ return 0;
+ } else if (null == this.keeper) {
+ return -1;
+ } else if (null == other.keeper) {
+ return 1;
+ } else {
+ return this.keeper.compareTo(other.keeper);
+ }
+ }
+ }
+ }
+
+ private static SortedSet<ZooKeeperState> status = new TreeSet<ZooKeeperState>();
+
+ public static Collection<ZooKeeperState> getZooKeeperStatus() {
+ return status;
+ }
+
+ public void stop() {
+ this.stop = true;
+ }
+
+ @Override
+ public void run() {
+
+ while (!stop) {
+
+ TreeSet<ZooKeeperState> update = new TreeSet<ZooKeeperState>();
+
+ String zookeepers[] = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST).split(",");
+ for (String keeper : zookeepers) {
+ int clients = 0;
+ String mode = "unknown";
+
+ String[] parts = keeper.split(":");
+ TTransport transport = null;
+ try {
+ HostAndPort addr;
+ if (parts.length > 1)
+ addr = HostAndPort.fromParts(parts[0], Integer.parseInt(parts[1]));
+ else
+ addr = HostAndPort.fromParts(parts[0], 2181);
+
+ transport = TTimeoutTransport.create(addr, 10 * 1000l);
- transport.write("stat\n".getBytes(Constants.UTF8), 0, 5);
++ transport.write("stat\n".getBytes(UTF_8), 0, 5);
+ StringBuilder response = new StringBuilder();
+ try {
+ transport.flush();
+ byte[] buffer = new byte[1024 * 100];
+ int n = 0;
+ while ((n = transport.read(buffer, 0, buffer.length)) > 0) {
- response.append(new String(buffer, 0, n, Constants.UTF8));
++ response.append(new String(buffer, 0, n, UTF_8));
+ }
+ } catch (TTransportException ex) {
+ // happens at EOF
+ }
+ for (String line : response.toString().split("\n")) {
+ if (line.startsWith(" "))
+ clients++;
+ if (line.startsWith("Mode"))
+ mode = line.split(":")[1];
+ }
+ update.add(new ZooKeeperState(keeper, mode, clients));
+ } catch (Exception ex) {
+ log.info("Exception talking to zookeeper " + keeper, ex);
+ update.add(new ZooKeeperState(keeper, "Down", -1));
+ } finally {
+ if (transport != null) {
+ try {
+ transport.close();
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }
+ status = update;
+ UtilWaitThread.sleep(5 * 1000);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index d896040,0000000..7812603
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@@ -1,283 -1,0 +1,285 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.util.Date;
+import java.util.List;
+import java.util.TimerTask;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.monitor.DedupedLogEvent;
+import org.apache.accumulo.server.monitor.LogService;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+abstract public class BasicServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 1L;
+ protected static final Logger log = Logger.getLogger(BasicServlet.class);
+ static String cachedInstanceName = null;
+ private static String bannerText;
+ private static String bannerColor;
+ private static String bannerBackground;
+
+ abstract protected String getTitle(HttpServletRequest req);
+
+ @Override
+ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ StringBuilder sb = new StringBuilder();
+ try {
+ Monitor.fetchData();
+ bannerText = sanitize(Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_TEXT));
+ bannerColor = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_COLOR).replace("'", "'");
+ bannerBackground = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_BACKGROUND).replace("'", "'");
+ pageStart(req, resp, sb);
+ pageBody(req, resp, sb);
+ pageEnd(req, resp, sb);
+ } catch (Throwable t) {
+ log.error("Error building page " + req.getRequestURI(), t);
+ sb.append("\n<pre>\n");
+ StringWriter sw = new StringWriter();
+ t.printStackTrace(new PrintWriter(sw));
+ sb.append(sanitize(sw.getBuffer().toString()));
+ sb.append("</pre>\n");
+ } finally {
+ resp.getWriter().print(sb);
+ resp.getWriter().flush();
+ }
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ doGet(req, resp);
+ }
+
+ private static final String DEFAULT_CONTENT_TYPE = "text/html";
+
+ public static final void setCookie(HttpServletResponse resp, String name, String value) {
+ resp.addCookie(new Cookie(name, value));
+ }
+
+ public static final String getCookieValue(HttpServletRequest req, String name) {
+ if (req.getCookies() != null)
+ for (Cookie c : req.getCookies())
+ if (c.getName().equals(name))
+ return c.getValue();
+ return null;
+ }
+
+ protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
+ resp.setContentType(DEFAULT_CONTENT_TYPE);
+ int refresh = -1;
+ String refreshStr = getCookieValue(req, "page.refresh.rate");
+ if (refreshStr != null) {
+ try {
+ refresh = Integer.parseInt(BasicServlet.decode(refreshStr));
+ } catch (NumberFormatException e) {
+ // ignore improperly formatted user cookie
+ }
+ }
+ synchronized (BasicServlet.class) {
+ // Learn our instance name asynchronously so we don't hang up if zookeeper is down
+ if (cachedInstanceName == null) {
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ synchronized (BasicServlet.class) {
+ if (cachedInstanceName == null) {
+ cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName();
+ }
+ }
+ }
+ }, 1000);
+ }
+ }
+
+ // BEGIN PAGE
+ sb.append("<!--\n");
+ sb.append(" Licensed to the Apache Software Foundation (ASF) under one or more\n");
+ sb.append(" contributor license agreements. See the NOTICE file distributed with\n");
+ sb.append(" this work for additional information regarding copyright ownership.\n");
+ sb.append(" The ASF licenses this file to You under the Apache License, Version 2.0\n");
+ sb.append(" (the \"License\"); you may not use this file except in compliance with\n");
+ sb.append(" the License. You may obtain a copy of the License at\n");
+ sb.append("\n");
+ sb.append(" http://www.apache.org/licenses/LICENSE-2.0\n");
+ sb.append("\n");
+ sb.append(" Unless required by applicable law or agreed to in writing, software\n");
+ sb.append(" distributed under the License is distributed on an \"AS IS\" BASIS,\n");
+ sb.append(" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n");
+ sb.append(" See the License for the specific language governing permissions and\n");
+ sb.append(" limitations under the License.\n");
+ sb.append("-->\n");
+ sb.append("<html>\n");
+
+ // BEGIN HEADER
+ sb.append("<head>\n");
+ sb.append("<title>").append(getTitle(req)).append(" - Accumulo ").append(Constants.VERSION).append("</title>\n");
+ if ((refresh > 0) && (req.getRequestURI().startsWith("/vis") == false) && (req.getRequestURI().startsWith("/shell") == false))
+ sb.append("<meta http-equiv='refresh' content='" + refresh + "' />\n");
+ sb.append("<meta http-equiv='Content-Type' content='").append(DEFAULT_CONTENT_TYPE).append("' />\n");
+ sb.append("<meta http-equiv='Content-Script-Type' content='text/javascript' />\n");
+ sb.append("<meta http-equiv='Content-Style-Type' content='text/css' />\n");
+ sb.append("<link rel='shortcut icon' type='image/jpg' href='/web/favicon.png' />\n");
+ sb.append("<link rel='stylesheet' type='text/css' href='/web/screen.css' media='screen' />\n");
+ sb.append("<script src='/web/functions.js' type='text/javascript'></script>\n");
+
+ sb.append("<!--[if lte IE 8]><script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/excanvas.min.js\"></script><![endif]-->\n");
+ sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.js\"></script>\n");
+ sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.flot.js\"></script>\n");
+
+ sb.append("</head>\n");
+
+ // BEGIN BODY OPENING
+ sb.append("\n<body ").append(getBodyAttributes()).append(">\n");
+ sb.append("<div id='content-wrapper'>\n");
+ sb.append("<div id='content'>\n");
+ sb.append("<div id='header'>");
+ if (!bannerText.isEmpty()) {
+ sb.append("<div id='banner' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
+ .append("</div>\n");
+ }
+ sb.append("<div id='headertitle'>");
+ sb.append("<h1>").append(getTitle(req)).append("</h1></div>\n");
+ sb.append("<div id='subheader'>Instance Name: ").append(cachedInstanceName).append(" Version: ").append(Constants.VERSION)
+ .append("\n");
+ sb.append("<br><span class='smalltext'>Instance ID: ").append(HdfsZooInstance.getInstance().getInstanceID()).append("</span>\n");
+ sb.append("<br><span class='smalltext'>").append(new Date().toString().replace(" ", " ")).append("</span>");
+ sb.append("</div>\n"); // end <div id='subheader'>
+ sb.append("</div>\n"); // end <div id='header'>
+
+ // BEGIN LEFT SIDE
+ sb.append("<div id='nav'>\n");
+ sb.append("<span id='nav-title'><a href='/'>Overview</a></span><br />\n");
+ sb.append("<hr />\n");
+ sb.append("<a href='/master'>Master Server</a><br />\n");
+ sb.append("<a href='/tservers'>Tablet Servers</a><br />\n");
+ sb.append("<a href='/scans'>Active Scans</a><br />\n");
+ sb.append("<a href='/vis'>Server Activity</a><br />\n");
+ sb.append("<a href='/gc'>Garbage Collector</a><br />\n");
+ sb.append("<a href='/tables'>Tables</a><br />\n");
+ sb.append("<a href='/trace/summary?minutes=10'>Recent Traces</a><br />\n");
+ List<DedupedLogEvent> dedupedLogEvents = LogService.getInstance().getEvents();
+ int numLogs = dedupedLogEvents.size();
+ boolean logsHaveError = false;
+ for (DedupedLogEvent dedupedLogEvent : dedupedLogEvents)
+ if (dedupedLogEvent.getEvent().getLevel().isGreaterOrEqual(Level.ERROR)) {
+ logsHaveError = true;
+ break;
+ }
+ if (numLogs > 0)
+ sb.append("<span class='" + (logsHaveError ? "error" : "warning") + "'><a href='/log'>Recent Logs <span class='smalltext'>(" + numLogs
+ + ")</a></span></span><br />\n");
+ int numProblems = Monitor.getProblemSummary().entrySet().size();
+ if (numProblems > 0)
+ sb.append("<span class='error'><a href='/problems'>Table Problems <span class='smalltext'>(" + numProblems + ")</a></span></span><br />\n");
+ sb.append("<hr />\n");
+ sb.append("<a href='/xml'>XML</a><br />\n");
+ sb.append("<a href='/json'>JSON</a><hr />\n");
+ if (Monitor.isUsingSsl())
+ sb.append("<a href='/shell'>Shell</a><hr />\n");
+ sb.append("<div class='smalltext'>[<a href='").append("/op?action=refresh&value=").append(refresh < 1 ? "5" : "-1");
+ sb.append("&redir=").append(currentPage(req)).append("'>");
+ sb.append(refresh < 1 ? "en" : "dis").append("able auto-refresh</a>]</div>\n");
+ sb.append("</div>\n"); // end <div id='nav'>
+
+ sb.append("<div id='main'");
+ if (bannerText.isEmpty())
+ sb.append(" style='bottom:0'");
+ sb.append(">\n");
+ sb.append("<!-- BEGIN MAIN BODY CONTENT -->\n\n");
+ }
+
+ protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
+ sb.append("This page intentionally left blank.");
+ }
+
+ protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
+ sb.append("\n<!-- END MAIN BODY CONTENT -->\n");
+ sb.append("</div>\n"); // end <div id='main'>
+
+ // BEGIN FOOTER
+ sb.append("</div>\n"); // end <div id='content'>
+ sb.append("</div>\n"); // end <div id='content-wrapper'>
+ if (!bannerText.isEmpty()) {
+ sb.append("<div id='footer' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
+ .append("</div>\n");
+ }
+ sb.append("</body>\n");
+ sb.append("</html>\n");
+ }
+
+ /**
+ * Allow the concrete servlet implementation to provide attributes on the body HTML tag, such as 'onload', which can be used to call Javascript methods on
+ * page load. By default, nothing is specified.
+ */
+ protected String getBodyAttributes() {
+ return "";
+ }
+
+ public static String encode(String s) {
+ try {
- return URLEncoder.encode(s, Constants.UTF8.name());
++ return URLEncoder.encode(s, UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
- Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++ Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
+ throw new AssertionError(e); // can't happen with UTF-8
+ }
+ }
+
+ public static String decode(String s) {
+ try {
- return URLDecoder.decode(s, Constants.UTF8.name());
++ return URLDecoder.decode(s, UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
- Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++ Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
+ throw new AssertionError(e); // can't happen with UTF-8
+ }
+ }
+
+ public static String sanitize(String xml) {
+ return xml.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
+ }
+
+ public static String currentPage(HttpServletRequest req) {
+ String redir = req.getRequestURI();
+ if (req.getQueryString() != null)
+ redir += "?" + req.getQueryString();
+ return encode(redir);
+ }
+
+ protected static void banner(StringBuilder sb, String klass, String text) {
+ sb.append("<br />\n<h2 class='").append(klass).append("'>").append(text).append("</h2>\n");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index d88bd7c,0000000..bb7e690
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -1,346 -1,0 +1,347 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.util.Duration;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.ZooKeeperStatus;
+import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
+import org.apache.accumulo.monitor.util.celltypes.NumberType;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class DefaultServlet extends BasicServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "Accumulo Overview";
+ }
+
+ private void getResource(HttpServletRequest req, HttpServletResponse resp) throws IOException {
+ try {
+ String path = req.getRequestURI();
+
+ if (path.endsWith(".jpg"))
+ resp.setContentType("image/jpeg");
+
+ if (path.endsWith(".html"))
+ resp.setContentType("text/html");
+
+ path = path.substring(1);
+ InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path);
+ ServletOutputStream out = resp.getOutputStream();
+ try {
+ if (data != null) {
+ byte[] buffer = new byte[1024];
+ int n;
+ while ((n = data.read(buffer)) > 0)
+ out.write(buffer, 0, n);
+ } else {
- out.write(("could not get resource " + path + "").getBytes(Constants.UTF8));
++ out.write(("could not get resource " + path + "").getBytes(UTF_8));
+ }
+ } finally {
+ if (data != null)
+ data.close();
+ }
+ } catch (Throwable t) {
+ log.error(t, t);
+ throw new IOException(t);
+ }
+ }
+
+ @Override
+ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ if (req.getRequestURI().startsWith("/web"))
+ getResource(req, resp);
+ else if (req.getRequestURI().startsWith("/monitor"))
+ resp.sendRedirect("/master");
+ else if (req.getRequestURI().startsWith("/errors"))
+ resp.sendRedirect("/problems");
+ else
+ super.doGet(req, resp);
+ }
+
+ public static final int GRAPH_WIDTH = 450;
+ public static final int GRAPH_HEIGHT = 150;
+
+ private static void plotData(StringBuilder sb, String title, @SuppressWarnings("rawtypes") List data, boolean points) {
+ plotData(sb, title, points, new ArrayList<String>(), data);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static void plotData(StringBuilder sb, String title, boolean points, List<String> labels, List... series) {
+ sb.append("<div class=\"plotHeading\">");
+ sb.append(title);
+ sb.append("</div>");
+ sb.append("</br>");
+ String id = "c" + title.hashCode();
+ sb.append("<div id=\"" + id + "\" style=\"width:" + GRAPH_WIDTH + "px;height:" + GRAPH_HEIGHT + "px;\"></div>\n");
+
+ sb.append("<script type=\"text/javascript\">\n");
+ sb.append("$(function () {\n");
+
+ for (int i = 0; i < series.length; i++) {
+
+ @SuppressWarnings("unchecked")
+ List<Pair<Long,? extends Number>> data = series[i];
+ sb.append(" var d" + i + " = [");
+
+ String sep = "";
+ for (Pair<Long,? extends Number> point : data) {
+ if (point.getSecond() == null)
+ continue;
+
+ String y;
+ if (point.getSecond() instanceof Double)
+ y = String.format("%1.2f", point.getSecond());
+ else
+ y = point.getSecond().toString();
+
+ sb.append(sep);
+ sep = ",";
+ sb.append("[" + utc2local(point.getFirst()) + "," + y + "]");
+ }
+ sb.append(" ];\n");
+ }
+
+ String opts = "lines: { show: true }";
+ if (points)
+ opts = "points: { show: true, radius: 1 }";
+
+ sb.append(" $.plot($(\"#" + id + "\"),");
+ String sep = "";
+
+ String colors[] = new String[] {"red", "blue", "green", "black"};
+
+ sb.append("[");
+ for (int i = 0; i < series.length; i++) {
+ sb.append(sep);
+ sep = ",";
+ sb.append("{ ");
+ if (labels.size() > 0) {
+ sb.append("label: \"" + labels.get(i) + "\", ");
+ }
+ sb.append("data: d" + i + ", " + opts + ", color:\"" + colors[i] + "\" }");
+ }
+ sb.append("], ");
+ sb.append("{yaxis:{}, xaxis:{mode:\"time\",minTickSize: [1, \"minute\"],timeformat: \"%H:%M<br />" + getShortTZName() + "\", ticks:3}});");
+ sb.append(" });\n");
+ sb.append("</script>\n");
+ }
+
+ /**
+ * Shows the current time zone (based on the current time) short name
+ */
+ private static String getShortTZName() {
+ TimeZone tz = TimeZone.getDefault();
+ return tz.getDisplayName(tz.inDaylightTime(new Date()), TimeZone.SHORT);
+ }
+
+ /**
+ * Converts a unix timestamp in UTC to one that is relative to the local timezone
+ */
+ private static Long utc2local(Long utcMillis) {
+ Calendar currentCalendar = Calendar.getInstance(); // default timezone
+ currentCalendar.setTimeInMillis(utcMillis + currentCalendar.getTimeZone().getOffset(utcMillis));
+ return currentCalendar.getTime().getTime();
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws IOException {
+
+ sb.append("<table class='noborder'>\n");
+ sb.append("<tr>\n");
+
+ sb.append("<td class='noborder'>\n");
+ doAccumuloTable(sb);
+ sb.append("</td>\n");
+
+ sb.append("<td class='noborder'>\n");
+ doZooKeeperTable(sb);
+ sb.append("</td>\n");
+
+ sb.append("</tr></table>\n");
+ sb.append("<br/>\n");
+
+ sb.append("<p/><table class=\"noborder\">\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Ingest (Entries/s)", Monitor.getIngestRateOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Scan (Entries/s)", false, Arrays.asList("Read", "Returned"), Monitor.getScanRateOverTime(), Monitor.getQueryRateOverTime());
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Ingest (MB/s)", Monitor.getIngestByteRateOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Scan (MB/s)", Monitor.getQueryByteRateOverTime(), false);
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Load Average", Monitor.getLoadOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Seeks", Monitor.getLookupsOverTime(), false);
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Minor Compactions", Monitor.getMinorCompactionsOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Major Compactions", Monitor.getMajorCompactionsOverTime(), false);
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Index Cache Hit Rate", Monitor.getIndexCacheHitRateOverTime(), true);
+ sb.append("</td><td>\n");
+ plotData(sb, "Data Cache Hit Rate", Monitor.getDataCacheHitRateOverTime(), true);
+ sb.append("</td></tr>\n");
+
+ sb.append("</table>\n");
+ }
+
+ private void doAccumuloTable(StringBuilder sb) throws IOException {
+ // Accumulo
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
+ MasterMonitorInfo info = Monitor.getMmi();
+ sb.append("<table>\n");
+ sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
+ if (info == null) {
+ sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
+ } else {
+ long totalAcuBytesUsed = 0l;
+ long totalHdfsBytesUsed = 0l;
+
+ try {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
+ final Path basePath = new Path(baseDir);
+ final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
+
+ try {
+ // Calculate the amount of space used by Accumulo on the FileSystem
+ ContentSummary accumuloSummary = fs.getContentSummary(basePath);
+ long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed();
+ totalAcuBytesUsed += bytesUsedByAcuOnFs;
+
+ // Catch the overflow -- this is big data
+ if (totalAcuBytesUsed < bytesUsedByAcuOnFs) {
+ log.debug("Overflowed long in bytes used by Accumulo for " + baseDir);
+ totalAcuBytesUsed = 0l;
+ break;
+ }
+
+ // Calculate the total amount of space used on the FileSystem
+ ContentSummary volumeSummary = fs.getContentSummary(new Path("/"));
+ long bytesUsedOnVolume = volumeSummary.getSpaceConsumed();
+ totalHdfsBytesUsed += bytesUsedOnVolume;
+
+ // Catch the overflow -- this is big data
+ if (totalHdfsBytesUsed < bytesUsedOnVolume) {
+ log.debug("Overflowed long in bytes used in HDFS for " + baseDir);
+ totalHdfsBytesUsed = 0;
+ break;
+ }
+ } catch (Exception ex) {
+ log.trace("Unable to get disk usage information for " + baseDir, ex);
+ }
+ }
+
+ String diskUsed = "Unknown";
+ String consumed = null;
+ if (totalAcuBytesUsed > 0) {
+ // Convert Accumulo usage to a readable String
+ diskUsed = bytes(totalAcuBytesUsed);
+
+ if (totalHdfsBytesUsed > 0) {
+ // Compute amount of space used by Accumulo as a percentage of total space usage.
+ consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed);
+ }
+ }
+
+ boolean highlight = false;
+ tableRow(sb, (highlight = !highlight), "Disk Used", diskUsed);
+ if (null != consumed)
+ tableRow(sb, (highlight = !highlight), "% of Used DFS", consumed);
+ tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables()));
+ tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));
+ tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Dead Tablet Servers</a>", NumberType.commas(info.deadTabletServers.size(), 0, 0));
+ tableRow(sb, (highlight = !highlight), "Tablets", NumberType.commas(Monitor.getTotalTabletCount(), 1, Long.MAX_VALUE));
+ tableRow(sb, (highlight = !highlight), "Entries", NumberType.commas(Monitor.getTotalEntries()));
+ tableRow(sb, (highlight = !highlight), "Lookups", NumberType.commas(Monitor.getTotalLookups()));
+ tableRow(sb, (highlight = !highlight), "Uptime", Duration.format(System.currentTimeMillis() - Monitor.getStartTime()));
+ } catch (Exception e) {
+ log.debug(e, e);
+ }
+ }
+ sb.append("</table>\n");
+ }
+
+ private void doZooKeeperTable(StringBuilder sb) throws IOException {
+ // Zookeepers
+ sb.append("<table>\n");
+ sb.append("<tr><th colspan='3'>Zookeeper</th></tr>\n");
+ sb.append("<tr><th>Server</th><th>Mode</th><th>Clients</th></tr>\n");
+
+ boolean highlight = false;
+ for (ZooKeeperState k : ZooKeeperStatus.getZooKeeperStatus()) {
+ if (k.clients >= 0) {
+ tableRow(sb, (highlight = !highlight), k.keeper, k.mode, k.clients);
+ } else {
+ tableRow(sb, false, k.keeper, "<span class='error'>Down</span>", "");
+ }
+ }
+ sb.append("</table>\n");
+ }
+
+ private static String bytes(long big) {
+ return NumUtil.bigNumberForSize(big);
+ }
+
+ public static void tableRow(StringBuilder sb, boolean highlight, Object... cells) {
+ sb.append(highlight ? "<tr class='highlight'>" : "<tr>");
+ for (int i = 0; i < cells.length; ++i) {
+ Object cell = cells[i];
+ String cellValue = cell == null ? "" : String.valueOf(cell).trim();
+ sb.append("<td class='").append(i < cells.length - 1 ? "left" : "right").append("'>").append(cellValue.isEmpty() ? "-" : cellValue).append("</td>");
+ }
+ sb.append("</tr>\n");
+ }
+}
[16/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 1e7658a,0000000..6c40094
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -1,329 -1,0 +1,331 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.Version;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.ReadOnlyStore;
+import org.apache.accumulo.fate.ReadOnlyTStore;
+import org.apache.accumulo.fate.ZooStore;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.watcher.Log4jConfiguration;
+import org.apache.accumulo.server.watcher.MonitorLog4jWatcher;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.log4j.helpers.LogLog;
+import org.apache.log4j.xml.DOMConfigurator;
+import org.apache.zookeeper.KeeperException;
+
+public class Accumulo {
+
+ private static final Logger log = Logger.getLogger(Accumulo.class);
+
+ public static synchronized void updateAccumuloVersion(VolumeManager fs, int oldVersion) {
+ for (Volume volume : fs.getVolumes()) {
+ try {
+ if (getAccumuloPersistentVersion(fs) == oldVersion) {
+ log.debug("Attempting to upgrade " + volume);
+ Path dataVersionLocation = ServerConstants.getDataVersionLocation(volume);
+ fs.create(new Path(dataVersionLocation, Integer.toString(ServerConstants.DATA_VERSION))).close();
+ // TODO document failure mode & recovery if FS permissions cause above to work and below to fail ACCUMULO-2596
+ Path prevDataVersionLoc = new Path(dataVersionLocation, Integer.toString(oldVersion));
+ if (!fs.delete(prevDataVersionLoc)) {
+ throw new RuntimeException("Could not delete previous data version location (" + prevDataVersionLoc + ") for " + volume);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
+ }
+ }
+ }
+
+ public static synchronized int getAccumuloPersistentVersion(FileSystem fs, Path path) {
+ int dataVersion;
+ try {
+ FileStatus[] files = fs.listStatus(path);
+ if (files == null || files.length == 0) {
+ dataVersion = -1; // assume it is 0.5 or earlier
+ } else {
+ dataVersion = Integer.parseInt(files[0].getPath().getName());
+ }
+ return dataVersion;
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to read accumulo version: an error occurred.", e);
+ }
+ }
+
+ public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
+ // It doesn't matter which Volume is used as they should all have the data version stored
+ Volume v = fs.getVolumes().iterator().next();
+ Path path = ServerConstants.getDataVersionLocation(v);
+ return getAccumuloPersistentVersion(v.getFileSystem(), path);
+ }
+
+ public static synchronized Path getAccumuloInstanceIdPath(VolumeManager fs) {
+ // It doesn't matter which Volume is used as they should all have the instance ID stored
+ Volume v = fs.getVolumes().iterator().next();
+ return ServerConstants.getInstanceIdLocation(v);
+ }
+
+ public static void enableTracing(String address, String application) {
+ try {
+ DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address);
+ } catch (Exception ex) {
+ log.error("creating remote sink for trace spans", ex);
+ }
+ }
+
+ /**
+ * Finds the best log4j configuration file. A generic file is used only if an
+ * application-specific file is not available. An XML file is preferred over
+ * a properties file, if possible.
+ *
+ * @param confDir directory where configuration files should reside
+ * @param application application name for configuration file name
+ * @return configuration file name
+ */
+ static String locateLogConfig(String confDir, String application) {
+ String explicitConfigFile = System.getProperty("log4j.configuration");
+ if (explicitConfigFile != null) {
+ return explicitConfigFile;
+ }
+ String[] configFiles = {
+ String.format("%s/%s_logger.xml", confDir, application),
+ String.format("%s/%s_logger.properties", confDir, application),
+ String.format("%s/generic_logger.xml", confDir),
+ String.format("%s/generic_logger.properties", confDir)
+ };
+ String defaultConfigFile = configFiles[2]; // generic_logger.xml
+ for (String f : configFiles) {
+ if (new File(f).exists()) {
+ return f;
+ }
+ }
+ return defaultConfigFile;
+ }
+
+ public static void setupLogging(String application) throws UnknownHostException {
+ System.setProperty("org.apache.accumulo.core.application", application);
+
+ if (System.getenv("ACCUMULO_LOG_DIR") != null)
+ System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_LOG_DIR"));
+ else
+ System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_HOME") + "/logs/");
+
+ String localhost = InetAddress.getLocalHost().getHostName();
+ System.setProperty("org.apache.accumulo.core.ip.localhost.hostname", localhost);
+
+ // Use a specific log config, if it exists
+ String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
+ // Turn off messages about not being able to reach the remote logger... we protect against that.
+ LogLog.setQuietMode(true);
+
+ // Read the auditing config
+ String auditConfig = String.format("%s/auditLog.xml", System.getenv("ACCUMULO_CONF_DIR"));
+
+ // Set up local file-based logging right away
+ Log4jConfiguration logConf = new Log4jConfiguration(logConfigFile);
+ logConf.resetLogger();
+
+ // Watch the auditLog.xml for the future updates
+ DOMConfigurator.configureAndWatch(auditConfig, 5000);
+ }
+
+ public static void init(VolumeManager fs, ServerConfiguration serverConfig, String application) throws IOException {
+ final AccumuloConfiguration conf = serverConfig.getConfiguration();
+ final Instance instance = serverConfig.getInstance();
+
+ // Use a specific log config, if it exists
+ final String logConfigFile = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
+
+ // Set up polling log4j updates and log-forwarding using information advertised in zookeeper by the monitor
+ MonitorLog4jWatcher logConfigWatcher = new MonitorLog4jWatcher(instance.getInstanceID(), logConfigFile);
+ logConfigWatcher.setDelay(5000L);
+ logConfigWatcher.start();
+
+ // Makes sure the log-forwarding to the monitor is configured
+ int logPort = conf.getPort(Property.MONITOR_LOG4J_PORT);
+ System.setProperty("org.apache.accumulo.core.host.log.port", Integer.toString(logPort));
+
+ log.info(application + " starting");
+ log.info("Instance " + serverConfig.getInstance().getInstanceID());
+ int dataVersion = Accumulo.getAccumuloPersistentVersion(fs);
+ log.info("Data Version " + dataVersion);
+ Accumulo.waitForZookeeperAndHdfs(fs);
+
+ Version codeVersion = new Version(Constants.VERSION);
+ if (!(canUpgradeFromDataVersion(dataVersion))) {
+ throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible with files stored using data version " + dataVersion);
+ }
+
+ TreeMap<String,String> sortedProps = new TreeMap<String,String>();
+ for (Entry<String,String> entry : conf)
+ sortedProps.put(entry.getKey(), entry.getValue());
+
+ for (Entry<String,String> entry : sortedProps.entrySet()) {
+ String key = entry.getKey();
+ log.info(key + " = " + (Property.isSensitive(key) ? "<hidden>" : entry.getValue()));
+ }
+
+ monitorSwappiness();
+ }
+
+ /**
+ * Sanity check that the current persistent version is allowed to upgrade to the version of Accumulo running.
+ * @param dataVersion the version that is persisted in the backing Volumes
+ */
+ public static boolean canUpgradeFromDataVersion(final int dataVersion) {
+ return dataVersion == ServerConstants.DATA_VERSION || dataVersion == ServerConstants.PREV_DATA_VERSION || dataVersion == ServerConstants.TWO_DATA_VERSIONS_AGO;
+ }
+
+ /**
+ * Does the data version number stored in the backing Volumes indicate we need to upgrade something?
+ */
+ public static boolean persistentVersionNeedsUpgrade(final int accumuloPersistentVersion) {
+ return accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO || accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION;
+ }
+
+ /**
+ *
+ */
+ public static void monitorSwappiness() {
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ String procFile = "/proc/sys/vm/swappiness";
+ File swappiness = new File(procFile);
+ if (swappiness.exists() && swappiness.canRead()) {
+ InputStream is = new FileInputStream(procFile);
+ try {
+ byte[] buffer = new byte[10];
+ int bytes = is.read(buffer);
- String setting = new String(buffer, 0, bytes, Constants.UTF8);
++ String setting = new String(buffer, 0, bytes, UTF_8);
+ setting = setting.trim();
+ if (bytes > 0 && Integer.parseInt(setting) > 10) {
+ log.warn("System swappiness setting is greater than ten (" + setting + ") which can cause time-sensitive operations to be delayed. "
+ + " Accumulo is time sensitive because it needs to maintain distributed lock agreement.");
+ }
+ } finally {
+ is.close();
+ }
+ }
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+ }
+ }, 1000, 10 * 60 * 1000);
+ }
+
+ public static void waitForZookeeperAndHdfs(VolumeManager fs) {
+ log.info("Attempting to talk to zookeeper");
+ while (true) {
+ try {
+ ZooReaderWriter.getInstance().getChildren(Constants.ZROOT);
+ break;
+ } catch (InterruptedException e) {
+ // ignored
+ } catch (KeeperException ex) {
+ log.info("Waiting for accumulo to be initialized");
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
+ long sleep = 1000;
+ int unknownHostTries = 3;
+ while (true) {
+ try {
+ if (fs.isReady())
+ break;
+ log.warn("Waiting for the NameNode to leave safemode");
+ } catch (IOException ex) {
+ log.warn("Unable to connect to HDFS", ex);
+ } catch (IllegalArgumentException exception) {
+ /* Unwrap the UnknownHostException so we can deal with it directly */
+ if (exception.getCause() instanceof UnknownHostException) {
+ if (unknownHostTries > 0) {
+ log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause());
+ /* We need to make sure our sleep period is long enough to avoid getting a cached failure of the host lookup. */
+ sleep = Math.max(sleep, (AddressUtil.getAddressCacheNegativeTtl((UnknownHostException)(exception.getCause()))+1)*1000);
+ } else {
+ log.error("Unable to connect to HDFS and have exceeded max number of retries.", exception);
+ throw exception;
+ }
+ unknownHostTries--;
+ } else {
+ throw exception;
+ }
+ }
+ log.info("Backing off due to failure; current sleep period is " + sleep / 1000. + " seconds");
+ UtilWaitThread.sleep(sleep);
+ /* Back off to give transient failures more time to clear. */
+ sleep = Math.min(60 * 1000, sleep * 2);
+ }
+ log.info("Connected to HDFS");
+ }
+
+ /**
+ * Exit loudly if there are outstanding Fate operations.
+ * Since Fate serializes class names, we need to make sure there are no queued
+ * transactions from a previous version before continuing an upgrade. The status of the operations is
+ * irrelevant; those in SUCCESSFUL status cause the same problem as those just queued.
+ *
+ * Note that the Master should not allow write access to Fate until after all upgrade steps are complete.
+ *
+ * Should be called as a guard before performing any upgrade steps, after determining that an upgrade is needed.
+ *
+ * see ACCUMULO-2519
+ */
+ public static void abortIfFateTransactions() {
+ try {
+ final ReadOnlyTStore<Accumulo> fate = new ReadOnlyStore<Accumulo>(new ZooStore<Accumulo>(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZFATE,
+ ZooReaderWriter.getInstance()));
+ if (!(fate.list().isEmpty())) {
+ throw new AccumuloException("Aborting upgrade because there are outstanding FATE transactions from a previous Accumulo version. Please see the README document for instructions on what to do under your previous version.");
+ }
+ } catch (Exception exception) {
+ log.fatal("Problem verifying Fate readiness", exception);
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 3508164,0000000..05f3c6c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -1,198 -1,0 +1,200 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.client;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.ConnectorImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An implementation of Instance that looks in HDFS and ZooKeeper to find the master and root tablet location.
+ *
+ */
+public class HdfsZooInstance implements Instance {
+
+ private HdfsZooInstance() {
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ zooCache = new ZooCacheFactory().getZooCache(acuConf.get(Property.INSTANCE_ZK_HOST), (int) acuConf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
+ }
+
+ private static HdfsZooInstance cachedHdfsZooInstance = null;
+
+ public static synchronized Instance getInstance() {
+ if (cachedHdfsZooInstance == null)
+ cachedHdfsZooInstance = new HdfsZooInstance();
+ return cachedHdfsZooInstance;
+ }
+
+ private static ZooCache zooCache;
+ private static String instanceId = null;
+ private static final Logger log = Logger.getLogger(HdfsZooInstance.class);
+
+ @Override
+ public String getRootTabletLocation() {
+ String zRootLocPath = ZooUtil.getRoot(this) + RootTable.ZROOT_TABLET_LOCATION;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up root tablet location in zoocache.");
+
+ byte[] loc = zooCache.get(zRootLocPath);
+
- opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++ opTimer.stop("Found root tablet at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return null;
+ }
+
- return new String(loc, Constants.UTF8).split("\\|")[0];
++ return new String(loc, UTF_8).split("\\|")[0];
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+
+ String masterLocPath = ZooUtil.getRoot(this) + Constants.ZMASTER_LOCK;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Looking up master location in zoocache.");
+
+ byte[] loc = ZooLock.getLockData(zooCache, masterLocPath, null);
+
- opTimer.stop("Found master at " + (loc == null ? null : new String(loc, Constants.UTF8)) + " in %DURATION%");
++ opTimer.stop("Found master at " + (loc == null ? null : new String(loc, UTF_8)) + " in %DURATION%");
+
+ if (loc == null) {
+ return Collections.emptyList();
+ }
+
- return Collections.singletonList(new String(loc, Constants.UTF8));
++ return Collections.singletonList(new String(loc, UTF_8));
+ }
+
+ @Override
+ public String getInstanceID() {
+ if (instanceId == null)
+ _getInstanceID();
+ return instanceId;
+ }
+
+ private static synchronized void _getInstanceID() {
+ if (instanceId == null) {
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
+ log.trace("Looking for instanceId from " + instanceIdPath);
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
+ instanceId = instanceIdFromFile;
+ }
+ }
+
+ @Override
+ public String getInstanceName() {
+ return ZooKeeperInstance.lookupInstanceName(zooCache, UUID.fromString(getInstanceID()));
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return (int) ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ }
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ return new ConnectorImpl(this, new Credentials(principal, token));
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, new PasswordToken(pass));
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, ByteBufferUtil.toBytes(pass));
+ }
+
+ @Deprecated
+ @Override
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ return getConnector(user, TextUtil.getBytes(new Text(pass.toString())));
+ }
+
+ private AccumuloConfiguration conf = null;
+
+ @Deprecated
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ if (conf == null)
+ conf = new ServerConfiguration(this).getConfiguration();
+ return conf;
+ }
+
+ @Override
+ @Deprecated
+ public void setConfiguration(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ public static void main(String[] args) {
+ Instance instance = HdfsZooInstance.getInstance();
+ System.out.println("Instance Name: " + instance.getInstanceName());
+ System.out.println("Instance ID: " + instance.getInstanceID());
+ System.out.println("ZooKeepers: " + instance.getZooKeepers());
+ System.out.println("Masters: " + StringUtil.join(instance.getMasterLocations(), ", "));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
index df1be50,0000000..18feb4f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
@@@ -1,160 -1,0 +1,161 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.List;
+import java.util.Map;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration.PropertyFilter;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.log4j.Logger;
+
+/**
+ * A helper object for accessing properties in a {@link ZooCache}.
+ */
+public class ZooCachePropertyAccessor {
+ private static final Logger log = Logger.getLogger(ZooCachePropertyAccessor.class);
+
+ static class PropCacheKey {
+ final String instanceId;
+ final String scope;
+
+ PropCacheKey(String iid, String s) {
+ instanceId = iid;
+ scope = s;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof PropCacheKey)) {
+ return false;
+ }
+ PropCacheKey o = (PropCacheKey) other;
+ return (instanceId.equals(o.instanceId) && scope.equals(o.scope));
+ }
+
+ @Override
+ public int hashCode() {
+ int c = 17;
+ c = (37 * c) + instanceId.hashCode();
+ c = (37 * c) + scope.hashCode();
+ return c;
+ }
+ }
+
+ private final ZooCache propCache;
+
+ /**
+ * Creates a new accessor.
+ *
+ * @param propCache
+ * property cache
+ */
+ ZooCachePropertyAccessor(ZooCache propCache) {
+ this.propCache = propCache;
+ }
+
+ /**
+ * Gets the property cache accessed by this object.
+ *
+ * @return property cache
+ */
+ ZooCache getZooCache() {
+ return propCache;
+ }
+
+ /**
+ * Gets a property. If the property is not in ZooKeeper or is present but an invalid format for the property type, the parent configuration is consulted (if
+ * provided).
+ *
+ * @param property
+ * property to get
+ * @param path
+ * ZooKeeper path where properties lie
+ * @param parent
+ * parent configuration (optional)
+ * @return property value, or null if not found
+ */
+ String get(Property property, String path, AccumuloConfiguration parent) {
+ String key = property.getKey();
+ String value = get(path + "/" + key);
+
+ if (value == null || !property.getType().isValidFormat(value)) {
+ if (value != null) {
+ log.error("Using default value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
+ }
+ if (parent != null) {
+ value = parent.get(property);
+ }
+ }
+ return value;
+ }
+
+ private String get(String path) {
+ byte[] v = propCache.get(path);
+ if (v != null) {
- return new String(v, Constants.UTF8);
++ return new String(v, UTF_8);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Gets all properties into the given map. Properties are filtered using the given filter. Properties from a parent configuration are also added to the map
+ * and filtered, either using a separate filter or, if not specified, the other filter.
+ *
+ * @param props
+ * map to populate with properties
+ * @param path
+ * ZooKeeper path where properties lie
+ * @param filter
+ * property filter (required)
+ * @param parent
+ * parent configuration (required)
+ * @param parentFilter
+ * separate filter for parent properties (optional)
+ */
+ void getProperties(Map<String,String> props, String path, PropertyFilter filter, AccumuloConfiguration parent, PropertyFilter parentFilter) {
+ parent.getProperties(props, parentFilter != null ? parentFilter : filter);
+
+ List<String> children = propCache.getChildren(path);
+ if (children != null) {
+ for (String child : children) {
+ if (child != null && filter.accept(child)) {
+ String value = get(path + "/" + child);
+ if (value != null) {
+ props.put(child, value);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Clears the internal {@link ZooCache}.
+ */
+ void invalidateCache() {
+ propCache.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index a49ec43,0000000..6d25f1d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@@ -1,118 -1,0 +1,120 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.conf;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.log4j.Logger;
+
+public class ZooConfiguration extends AccumuloConfiguration {
+ private static final Logger log = Logger.getLogger(ZooConfiguration.class);
+
+ private final String instanceId;
+ private final ZooCache propCache;
+ private final AccumuloConfiguration parent;
+ private final Map<String,String> fixedProps = Collections.synchronizedMap(new HashMap<String,String>());
+
+ protected ZooConfiguration(String instanceId, ZooCache propCache, AccumuloConfiguration parent) {
+ this.instanceId = instanceId;
+ this.propCache = propCache;
+ this.parent = parent;
+ }
+
+ @Override
+ public void invalidateCache() {
+ if (propCache != null)
+ propCache.clear();
+ }
+
+ /**
+ * Gets the parent configuration of this configuration.
+ *
+ * @return parent configuration
+ */
+ public AccumuloConfiguration getParentConfiguration() {
+ return parent;
+ }
+
+ private String _get(Property property) {
+ String key = property.getKey();
+ String value = null;
+
+ if (Property.isValidZooPropertyKey(key)) {
+ value = get(key);
+ }
+
+ if (value == null || !property.getType().isValidFormat(value)) {
+ if (value != null)
+ log.error("Using parent value for " + key + " due to improperly formatted " + property.getType() + ": " + value);
+ value = parent.get(property);
+ }
+ return value;
+ }
+
+ @Override
+ public String get(Property property) {
+ if (Property.isFixedZooPropertyKey(property)) {
+ if (fixedProps.containsKey(property.getKey())) {
+ return fixedProps.get(property.getKey());
+ } else {
+ synchronized (fixedProps) {
+ String val = _get(property);
+ fixedProps.put(property.getKey(), val);
+ return val;
+ }
+
+ }
+ } else {
+ return _get(property);
+ }
+ }
+
+ private String get(String key) {
+ String zPath = ZooUtil.getRoot(instanceId) + Constants.ZCONFIG + "/" + key;
+ byte[] v = propCache.get(zPath);
+ String value = null;
+ if (v != null)
- value = new String(v, Constants.UTF8);
++ value = new String(v, UTF_8);
+ return value;
+ }
+
+ @Override
+ public void getProperties(Map<String,String> props, PropertyFilter filter) {
+ parent.getProperties(props, filter);
+
+ List<String> children = propCache.getChildren(ZooUtil.getRoot(instanceId) + Constants.ZCONFIG);
+ if (children != null) {
+ for (String child : children) {
+ if (child != null && filter.accept(child)) {
+ String value = get(child);
+ if (value != null)
+ props.put(child, value);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 807e821,0000000..587e38a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -1,315 -1,0 +1,317 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.constraints;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetadataConstraints implements Constraint {
+
+ private ZooCache zooCache = null;
+ private String zooRoot = null;
+
+ private static final Logger log = Logger.getLogger(MetadataConstraints.class);
+
+ private static boolean[] validTableNameChars = new boolean[256];
+
+ {
+ for (int i = 0; i < 256; i++) {
+ validTableNameChars[i] = ((i >= 'a' && i <= 'z') || (i >= '0' && i <= '9')) || i == '!';
+ }
+ }
+
+ private static final HashSet<ColumnFQ> validColumnQuals = new HashSet<ColumnFQ>(Arrays.asList(new ColumnFQ[] {
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN,
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN,
+ TabletsSection.ServerColumnFamily.LOCK_COLUMN, TabletsSection.ServerColumnFamily.FLUSH_COLUMN, TabletsSection.ServerColumnFamily.COMPACT_COLUMN}));
+
+ private static final HashSet<Text> validColumnFams = new HashSet<Text>(Arrays.asList(new Text[] {TabletsSection.BulkFileColumnFamily.NAME,
+ LogColumnFamily.NAME, ScanFileColumnFamily.NAME, DataFileColumnFamily.NAME,
+ TabletsSection.CurrentLocationColumnFamily.NAME, TabletsSection.LastLocationColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME,
+ ChoppedColumnFamily.NAME, ClonedColumnFamily.NAME}));
+
+ private static boolean isValidColumn(ColumnUpdate cu) {
+
+ if (validColumnFams.contains(new Text(cu.getColumnFamily())))
+ return true;
+
+ if (validColumnQuals.contains(new ColumnFQ(cu)))
+ return true;
+
+ return false;
+ }
+
+ static private ArrayList<Short> addViolation(ArrayList<Short> lst, int violation) {
+ if (lst == null)
+ lst = new ArrayList<Short>();
+ lst.add((short) violation);
+ return lst;
+ }
+
+ static private ArrayList<Short> addIfNotPresent(ArrayList<Short> lst, int intViolation) {
+ if (lst == null)
+ return addViolation(lst, intViolation);
+ short violation = (short) intViolation;
+ if (!lst.contains(violation))
+ return addViolation(lst, intViolation);
+ return lst;
+ }
+
+ @Override
+ public List<Short> check(Environment env, Mutation mutation) {
+
+ ArrayList<Short> violations = null;
+
+ Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
+
+ // check the row, it should contains at least one ; or end with <
+ boolean containsSemiC = false;
+
+ byte[] row = mutation.getRow();
+
+ // always allow rows that fall within reserved areas
+ if (row.length > 0 && row[0] == '~')
+ return null;
+ if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
+ return null;
+
+ for (byte b : row) {
+ if (b == ';') {
+ containsSemiC = true;
+ }
+
+ if (b == ';' || b == '<')
+ break;
+
+ if (!validTableNameChars[0xff & b]) {
+ violations = addIfNotPresent(violations, 4);
+ }
+ }
+
+ if (!containsSemiC) {
+ // see if last row char is <
+ if (row.length == 0 || row[row.length - 1] != '<') {
+ violations = addIfNotPresent(violations, 4);
+ }
+ } else {
+ if (row.length == 0) {
+ violations = addIfNotPresent(violations, 4);
+ }
+ }
+
+ if (row.length > 0 && row[0] == '!') {
+ if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
+ violations = addIfNotPresent(violations, 4);
+ }
+ }
+
+ // ensure row is not less than Constants.METADATA_TABLE_ID
+ if (new Text(row).compareTo(new Text(MetadataTable.ID)) < 0) {
+ violations = addViolation(violations, 5);
+ }
+
+ boolean checkedBulk = false;
+
+ for (ColumnUpdate columnUpdate : colUpdates) {
+ Text columnFamily = new Text(columnUpdate.getColumnFamily());
+
+ if (columnUpdate.isDeleted()) {
+ if (!isValidColumn(columnUpdate)) {
+ violations = addViolation(violations, 2);
+ }
+ continue;
+ }
+
+ if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
+ violations = addViolation(violations, 6);
+ }
+
+ if (columnFamily.equals(DataFileColumnFamily.NAME)) {
+ try {
+ DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
+
+ if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
+ violations = addViolation(violations, 1);
+ }
+ } catch (NumberFormatException nfe) {
+ violations = addViolation(violations, 1);
+ } catch (ArrayIndexOutOfBoundsException aiooe) {
+ violations = addViolation(violations, 1);
+ }
+ } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
+
+ } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
+ if (!columnUpdate.isDeleted() && !checkedBulk) {
+ // splits, which also write the time reference, are allowed to write this reference even when
+ // the transaction is not running because the other half of the tablet is holding a reference
+ // to the file.
+ boolean isSplitMutation = false;
+ // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
+ // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
+ // See ACCUMULO-1230.
+ boolean isLocationMutation = false;
+
+ HashSet<Text> dataFiles = new HashSet<Text>();
+ HashSet<Text> loadedFiles = new HashSet<Text>();
+
- String tidString = new String(columnUpdate.getValue(), Constants.UTF8);
++ String tidString = new String(columnUpdate.getValue(), UTF_8);
+ int otherTidCount = 0;
+
+ for (ColumnUpdate update : mutation.getUpdates()) {
+ if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
+ isSplitMutation = true;
+ } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
+ isLocationMutation = true;
+ } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
+ dataFiles.add(new Text(update.getColumnQualifier()));
+ } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
+ loadedFiles.add(new Text(update.getColumnQualifier()));
+
- if (!new String(update.getValue(), Constants.UTF8).equals(tidString)) {
++ if (!new String(update.getValue(), UTF_8).equals(tidString)) {
+ otherTidCount++;
+ }
+ }
+ }
+
+ if (!isSplitMutation && !isLocationMutation) {
+ long tid = Long.parseLong(tidString);
+
+ try {
+ if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
+ violations = addViolation(violations, 8);
+ }
+ } catch (Exception ex) {
+ violations = addViolation(violations, 8);
+ }
+ }
+
+ checkedBulk = true;
+ }
+ } else {
+ if (!isValidColumn(columnUpdate)) {
+ violations = addViolation(violations, 2);
+ } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0
+ && (violations == null || !violations.contains((short) 4))) {
+ KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
+
+ Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
+
+ boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
+
+ if (!prevEndRowLessThanEndRow) {
+ violations = addViolation(violations, 3);
+ }
+ } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
+ if (zooCache == null) {
+ zooCache = new ZooCache();
+ }
+
+ if (zooRoot == null) {
+ zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
+ }
+
+ boolean lockHeld = false;
- String lockId = new String(columnUpdate.getValue(), Constants.UTF8);
++ String lockId = new String(columnUpdate.getValue(), UTF_8);
+
+ try {
+ lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
+ } catch (Exception e) {
+ log.debug("Failed to verify lock was held " + lockId + " " + e.getMessage());
+ }
+
+ if (!lockHeld) {
+ violations = addViolation(violations, 7);
+ }
+ }
+
+ }
+ }
+
+ if (violations != null) {
- log.debug("violating metadata mutation : " + new String(mutation.getRow(), Constants.UTF8));
++ log.debug("violating metadata mutation : " + new String(mutation.getRow(), UTF_8));
+ for (ColumnUpdate update : mutation.getUpdates()) {
- log.debug(" update: " + new String(update.getColumnFamily(), Constants.UTF8) + ":" + new String(update.getColumnQualifier(), Constants.UTF8) + " value "
- + (update.isDeleted() ? "[delete]" : new String(update.getValue(), Constants.UTF8)));
++ log.debug(" update: " + new String(update.getColumnFamily(), UTF_8) + ":" + new String(update.getColumnQualifier(), UTF_8) + " value "
++ + (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
+ }
+ }
+
+ return violations;
+ }
+
+ protected Arbitrator getArbitrator() {
+ return new ZooArbitrator();
+ }
+
+ @Override
+ public String getViolationDescription(short violationCode) {
+ switch (violationCode) {
+ case 1:
+ return "data file size must be a non-negative integer";
+ case 2:
+ return "Invalid column name given.";
+ case 3:
+ return "Prev end row is greater than or equal to end row.";
+ case 4:
+ return "Invalid metadata row format";
+ case 5:
+ return "Row can not be less than " + MetadataTable.ID;
+ case 6:
+ return "Empty values are not allowed for any " + MetadataTable.NAME + " column";
+ case 7:
+ return "Lock not held in zookeeper by writer";
+ case 8:
+ return "Bulk load transaction no longer running";
+ }
+ return null;
+ }
+
+ @Override
+ protected void finalize() {
+ if (zooCache != null)
+ zooCache.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 24b5605,0000000..7c4b8d3
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@@ -1,586 -1,0 +1,587 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.init;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.constraints.MetadataConstraints;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * This class is used to setup the directory structure and the root tablet to get an instance started
+ *
+ */
+public class Initialize {
+ private static final Logger log = Logger.getLogger(Initialize.class);
+ private static final String DEFAULT_ROOT_USER = "root";
+ public static final String TABLE_TABLETS_TABLET_DIR = "/table_info";
+
+ private static ConsoleReader reader = null;
+ private static IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ private static ConsoleReader getConsoleReader() throws IOException {
+ if (reader == null)
+ reader = new ConsoleReader();
+ return reader;
+ }
+
+ /**
+ * Sets this class's ZooKeeper reader/writer.
+ *
+ * @param izoo
+ * reader/writer
+ */
+ static void setZooReaderWriter(IZooReaderWriter izoo) {
+ zoo = izoo;
+ }
+
+ /**
+ * Gets this class's ZooKeeper reader/writer.
+ *
+ * @return reader/writer
+ */
+ static IZooReaderWriter getZooReaderWriter() {
+ return zoo;
+ }
+
+ private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
+ static {
+ initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K");
+ initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
+ initialMetadataConf.put(Property.TABLE_WALOG_ENABLED.getKey(), "true");
+ initialMetadataConf.put(Property.TABLE_MAJC_RATIO.getKey(), "1");
+ initialMetadataConf.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "64M");
+ initialMetadataConf.put(Property.TABLE_CONSTRAINT_PREFIX.getKey() + "1", MetadataConstraints.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers", "10," + VersioningIterator.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers.opt.maxVersions", "1");
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers", "10," + VersioningIterator.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers.opt.maxVersions", "1");
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers", "10," + VersioningIterator.class.getName());
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers.opt.maxVersions", "1");
+ initialMetadataConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.bulkLoadFilter", "20," + MetadataBulkLoadFilter.class.getName());
+ initialMetadataConf.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "tablet",
+ String.format("%s,%s", TabletsSection.TabletColumnFamily.NAME, TabletsSection.CurrentLocationColumnFamily.NAME));
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + "server", String.format("%s,%s,%s,%s", TabletsSection.DataFileColumnFamily.NAME,
+ TabletsSection.LogColumnFamily.NAME, TabletsSection.ServerColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME));
+ initialMetadataConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(), "tablet,server");
+ initialMetadataConf.put(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
+ initialMetadataConf.put(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
+ initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
+ }
+
+ static boolean checkInit(Configuration conf, VolumeManager fs, SiteConfiguration sconf) throws IOException {
+ @SuppressWarnings("deprecation")
+ String fsUri = sconf.get(Property.INSTANCE_DFS_URI);
+ if (fsUri.equals(""))
+ fsUri = FileSystem.getDefaultUri(conf).toString();
+ log.info("Hadoop Filesystem is " + fsUri);
+ log.info("Accumulo data dirs are " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
+ log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
+ log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
+ if (!zookeeperAvailable()) {
+ log.fatal("Zookeeper needs to be up and running in order to init. Exiting ...");
+ return false;
+ }
+ if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) {
+ ConsoleReader c = getConsoleReader();
+ c.beep();
+ c.println();
+ c.println();
+ c.println("Warning!!! Your instance secret is still set to the default, this is not secure. We highly recommend you change it.");
+ c.println();
+ c.println();
+ c.println("You can change the instance secret in accumulo by using:");
+ c.println(" bin/accumulo " + org.apache.accumulo.server.util.ChangeSecret.class.getName() + " oldPassword newPassword.");
+ c.println("You will also need to edit your secret in your configuration file by adding the property instance.secret to your conf/accumulo-site.xml. Without this accumulo will not operate correctly");
+ }
+ try {
+ if (isInitialized(fs)) {
+ printInitializeFailureMessages(sconf);
+ return false;
+ }
+ } catch (IOException e) {
+ throw new IOException("Failed to check if filesystem already initialized", e);
+ }
+
+ return true;
+ }
+
+ static void printInitializeFailureMessages(SiteConfiguration sconf) {
+ @SuppressWarnings("deprecation")
+ Property INSTANCE_DFS_DIR = Property.INSTANCE_DFS_DIR;
+ @SuppressWarnings("deprecation")
+ Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI;
+ String instanceDfsDir = sconf.get(INSTANCE_DFS_DIR);
+ log.fatal("It appears the directories " + Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()))
+ + " were previously initialized.");
+ String instanceVolumes = sconf.get(Property.INSTANCE_VOLUMES);
+ String instanceDfsUri = sconf.get(INSTANCE_DFS_URI);
+
+ if (!instanceVolumes.isEmpty()) {
+ log.fatal("Change the property " + Property.INSTANCE_VOLUMES + " to use different filesystems,");
+ } else if (!instanceDfsDir.isEmpty()) {
+ log.fatal("Change the property " + INSTANCE_DFS_URI + " to use a different filesystem,");
+ } else {
+ log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_VOLUMES + " to use a different filesystem,");
+ }
+ log.fatal("or change the property " + INSTANCE_DFS_DIR + " to use a different directory.");
+ log.fatal("The current value of " + INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|");
+ log.fatal("The current value of " + INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|");
+ log.fatal("The current value of " + Property.INSTANCE_VOLUMES + " is |" + instanceVolumes + "|");
+ }
+
+ public static boolean doInit(Opts opts, Configuration conf, VolumeManager fs) throws IOException {
+ if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) {
+ return false;
+ }
+
+ // prompt user for instance name and root password early, in case they
+ // abort, we don't leave an inconsistent HDFS/ZooKeeper structure
+ String instanceNamePath;
+ try {
+ instanceNamePath = getInstanceNamePath(opts);
+ } catch (Exception e) {
+ log.fatal("Failed to talk to zookeeper", e);
+ return false;
+ }
+ opts.rootpass = getRootPassword(opts);
+ return initialize(opts, instanceNamePath, fs);
+ }
+
+ public static boolean initialize(Opts opts, String instanceNamePath, VolumeManager fs) {
+
+ UUID uuid = UUID.randomUUID();
+ // the actual disk locations of the root table and tablets
+ String[] configuredVolumes = VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration());
+ final String rootTabletDir = new Path(fs.choose(configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + RootTable.ID
+ + RootTable.ROOT_TABLET_LOCATION).toString();
+
+ try {
+ initZooKeeper(opts, uuid.toString(), instanceNamePath, rootTabletDir);
+ } catch (Exception e) {
+ log.fatal("Failed to initialize zookeeper", e);
+ return false;
+ }
+
+ try {
+ initFileSystem(opts, fs, uuid, rootTabletDir);
+ } catch (Exception e) {
+ log.fatal("Failed to initialize filesystem", e);
+
+ if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES).trim().equals("")) {
+ Configuration fsConf = CachedConfiguration.getInstance();
+
+ final String defaultFsUri = "file:///";
+ String fsDefaultName = fsConf.get("fs.default.name", defaultFsUri), fsDefaultFS = fsConf.get("fs.defaultFS", defaultFsUri);
+
+ // Try to determine when we couldn't find an appropriate core-site.xml on the classpath
+ if (defaultFsUri.equals(fsDefaultName) && defaultFsUri.equals(fsDefaultFS)) {
+ log.fatal("Default filesystem value ('fs.defaultFS' or 'fs.default.name') of '" + defaultFsUri + "' was found in the Hadoop configuration");
+ log.fatal("Please ensure that the Hadoop core-site.xml is on the classpath using 'general.classpaths' in accumulo-site.xml");
+ }
+ }
+
+ return false;
+ }
+
+ try {
+ initSecurity(opts, uuid.toString());
+ } catch (Exception e) {
+ log.fatal("Failed to initialize security", e);
+ return false;
+ }
+ return true;
+ }
+
+ private static boolean zookeeperAvailable() {
+ try {
+ return zoo.exists("/");
+ } catch (KeeperException e) {
+ return false;
+ } catch (InterruptedException e) {
+ return false;
+ }
+ }
+
+ private static void initDirs(VolumeManager fs, UUID uuid, String[] baseDirs, boolean print) throws IOException {
+ for (String baseDir : baseDirs) {
+ fs.mkdirs(new Path(new Path(baseDir, ServerConstants.VERSION_DIR), "" + ServerConstants.DATA_VERSION));
+
+ // create an instance id
+ Path iidLocation = new Path(baseDir, ServerConstants.INSTANCE_ID_DIR);
+ fs.mkdirs(iidLocation);
+ fs.createNewFile(new Path(iidLocation, uuid.toString()));
+ if (print)
+ log.info("Initialized volume " + baseDir);
+ }
+ }
+
+ private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, String rootTabletDir) throws IOException {
+ initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), false);
+
+ // initialize initial metadata config in zookeeper
+ initMetadataConfig();
+
+ String tableMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
+ + TABLE_TABLETS_TABLET_DIR;
+ String defaultMetadataTabletDir = fs.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID
+ + Constants.DEFAULT_TABLET_LOCATION;
+
+ // create table and default tablets directories
+ createDirectories(fs, rootTabletDir, tableMetadataTabletDir, defaultMetadataTabletDir);
+
+ // populate the root tablet with info about the metadata tablets
+ String fileName = rootTabletDir + Path.SEPARATOR + "00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
+ createMetadataFile(fs, fileName, MetadataTable.ID, tableMetadataTabletDir, defaultMetadataTabletDir);
+ }
+
+ /**
+ * Create an rfile in the default tablet's directory for a new table. This method is used to create the initial root tablet contents, with information about
+ * the metadata table's tablets
+ *
+ * @param volmanager
+ * The VolumeManager
+ * @param fileName
+ * The location to create the file
+ * @param tableId
+ * TableID that is being "created"
+ * @param tableTabletDir
+ * The table_info directory for the new table
+ * @param defaultTabletDir
+ * The default_tablet directory for the new table
+ */
+ private static void createMetadataFile(VolumeManager volmanager, String fileName, String tableId, String tableTabletDir, String defaultTabletDir)
+ throws IOException {
+ FileSystem fs = volmanager.getVolumeByPath(new Path(fileName)).getFileSystem();
+ FileSKVWriter tabletWriter = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
+ tabletWriter.startDefaultLocalityGroup();
+
+ Text splitPoint = TabletsSection.getRange().getEndKey().getRow();
+ createEntriesForTablet(tabletWriter, tableId, tableTabletDir, null, splitPoint);
+ createEntriesForTablet(tabletWriter, tableId, defaultTabletDir, splitPoint, null);
+
+ tabletWriter.close();
+ }
+
+ private static void createEntriesForTablet(FileSKVWriter writer, String tableId, String tabletDir, Text tabletPrevEndRow, Text tabletEndRow)
+ throws IOException {
+ Text extent = new Text(KeyExtent.getMetadataEntry(new Text(tableId), tabletEndRow));
- addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(Constants.UTF8)));
- addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(Constants.UTF8)));
++ addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(UTF_8)));
++ addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8)));
+ addEntry(writer, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tabletPrevEndRow));
+ }
+
+ private static void addEntry(FileSKVWriter writer, Text row, ColumnFQ col, Value value) throws IOException {
+ writer.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier(), 0), value);
+ }
+
+ private static void createDirectories(VolumeManager fs, String... dirs) throws IOException {
+ for (String s : dirs) {
+ Path dir = new Path(s);
+ try {
+ FileStatus fstat = fs.getFileStatus(dir);
+ // TODO Remove deprecation warning suppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ boolean isDirectory = fstat.isDir();
+ if (!isDirectory) {
+ log.fatal("location " + dir + " exists but is not a directory");
+ return;
+ }
+ } catch (FileNotFoundException fnfe) {
+ // attempt to create directory, since it doesn't exist
+ if (!fs.mkdirs(dir)) {
+ log.fatal("unable to create directory " + dir);
+ return;
+ }
+ }
+ }
+ }
+
+ private static void initZooKeeper(Opts opts, String uuid, String instanceNamePath, String rootTabletDir) throws KeeperException, InterruptedException {
+ // setup basic data in zookeeper
+ zoo.putPersistentData(Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
+ zoo.putPersistentData(Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
+
+ // setup instance name
+ if (opts.clearInstanceName)
+ zoo.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
- zoo.putPersistentData(instanceNamePath, uuid.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(instanceNamePath, uuid.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+
+ final byte[] EMPTY_BYTE_ARRAY = new byte[0], ZERO_CHAR_ARRAY = new byte[] {'0'};
+
+ // setup the instance
+ String zkInstanceRoot = Constants.ZROOT + "/" + uuid;
+ zoo.putPersistentData(zkInstanceRoot, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES, Constants.ZTABLES_INITIAL_ID, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES, new byte[0], NodeExistsPolicy.FAIL);
+ TableManager.prepareNewNamespaceState(uuid, Namespaces.DEFAULT_NAMESPACE_ID, Namespaces.DEFAULT_NAMESPACE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewNamespaceState(uuid, Namespaces.ACCUMULO_NAMESPACE_ID, Namespaces.ACCUMULO_NAMESPACE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, RootTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
+ TableManager.prepareNewTableState(uuid, MetadataTable.ID, Namespaces.ACCUMULO_NAMESPACE_ID, MetadataTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTSERVERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZPROBLEMS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_WALOGS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET_PATH, rootTabletDir.getBytes(UTF_8), NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(Constants.UTF8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8), NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLE_LOCKS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZHDFS_RESERVATIONS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZNEXT_FILE, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZRECOVERY, ZERO_CHAR_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZMONITOR_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ }
+
+ private static String getInstanceNamePath(Opts opts) throws IOException, KeeperException, InterruptedException {
+ // setup the instance name
+ String instanceName, instanceNamePath = null;
+ boolean exists = true;
+ do {
+ if (opts.cliInstanceName == null) {
+ instanceName = getConsoleReader().readLine("Instance name : ");
+ } else {
+ instanceName = opts.cliInstanceName;
+ }
+ if (instanceName == null)
+ System.exit(0);
+ instanceName = instanceName.trim();
+ if (instanceName.length() == 0)
+ continue;
+ instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName;
+ if (opts.clearInstanceName) {
+ exists = false;
+ break;
+ } else if (exists = zoo.exists(instanceNamePath)) {
+ String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : ");
+ if (decision == null)
+ System.exit(0);
+ if (decision.length() == 1 && decision.toLowerCase(Locale.ENGLISH).charAt(0) == 'y') {
+ opts.clearInstanceName = true;
+ exists = false;
+ }
+ }
+ } while (exists);
+ return instanceNamePath;
+ }
+
+ private static byte[] getRootPassword(Opts opts) throws IOException {
+ if (opts.cliPassword != null) {
- return opts.cliPassword.getBytes(Constants.UTF8);
++ return opts.cliPassword.getBytes(UTF_8);
+ }
+ String rootpass;
+ String confirmpass;
+ do {
+ rootpass = getConsoleReader()
+ .readLine("Enter initial password for " + DEFAULT_ROOT_USER + " (this may not be applicable for your security setup): ", '*');
+ if (rootpass == null)
+ System.exit(0);
+ confirmpass = getConsoleReader().readLine("Confirm initial password for " + DEFAULT_ROOT_USER + ": ", '*');
+ if (confirmpass == null)
+ System.exit(0);
+ if (!rootpass.equals(confirmpass))
+ log.error("Passwords do not match");
+ } while (!rootpass.equals(confirmpass));
- return rootpass.getBytes(Constants.UTF8);
++ return rootpass.getBytes(UTF_8);
+ }
+
+ private static void initSecurity(Opts opts, String iid) throws AccumuloSecurityException, ThriftSecurityException {
+ AuditedSecurityOperation.getInstance(iid, true).initializeSecurity(SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()), DEFAULT_ROOT_USER,
+ opts.rootpass);
+ }
+
+ public static void initMetadataConfig() throws IOException {
+ try {
+ Configuration conf = CachedConfiguration.getInstance();
+ int max = conf.getInt("dfs.replication.max", 512);
+ // Hadoop 0.23 switched the min value configuration name
+ int min = Math.max(conf.getInt("dfs.replication.min", 1), conf.getInt("dfs.namenode.replication.min", 1));
+ if (max < 5)
+ setMetadataReplication(max, "max");
+ if (min > 5)
+ setMetadataReplication(min, "min");
+ for (Entry<String,String> entry : initialMetadataConf.entrySet()) {
+ if (!TablePropUtil.setTableProperty(RootTable.ID, entry.getKey(), entry.getValue()))
+ throw new IOException("Cannot create per-table property " + entry.getKey());
+ if (!TablePropUtil.setTableProperty(MetadataTable.ID, entry.getKey(), entry.getValue()))
+ throw new IOException("Cannot create per-table property " + entry.getKey());
+ }
+ } catch (Exception e) {
+ log.fatal("error talking to zookeeper", e);
+ throw new IOException(e);
+ }
+ }
+
+ private static void setMetadataReplication(int replication, String reason) throws IOException {
+ String rep = getConsoleReader().readLine(
+ "Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME + " replication of 5. What do you want to set your "
+ + MetadataTable.NAME + " replication to? (" + replication + ") ");
+ if (rep == null || rep.length() == 0)
+ rep = Integer.toString(replication);
+ else
+ // Lets make sure it's a number
+ Integer.parseInt(rep);
+ initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), rep);
+ }
+
+ public static boolean isInitialized(VolumeManager fs) throws IOException {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
+ if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR)))
+ return true;
+ }
+
+ return false;
+ }
+
+ private static void addVolumes(VolumeManager fs) throws IOException {
+ HashSet<String> initializedDirs = new HashSet<String>();
+ initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
+
+ HashSet<String> uinitializedDirs = new HashSet<String>();
+ uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
+ uinitializedDirs.removeAll(initializedDirs);
+
+ Path aBasePath = new Path(initializedDirs.iterator().next());
+ Path iidPath = new Path(aBasePath, ServerConstants.INSTANCE_ID_DIR);
+ Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
+
+ UUID uuid = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(iidPath, ServerConfiguration.getSiteConfiguration()));
+
+ if (ServerConstants.DATA_VERSION != Accumulo.getAccumuloPersistentVersion(versionPath.getFileSystem(CachedConfiguration.getInstance()), versionPath)) {
+ throw new IOException("Accumulo " + Constants.VERSION + " cannot initialize data version " + Accumulo.getAccumuloPersistentVersion(fs));
+ }
+
+ initDirs(fs, uuid, uinitializedDirs.toArray(new String[uinitializedDirs.size()]), true);
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = "--add-volumes", description = "Initialize any uninitialized volumes listed in instance.volumes")
+ boolean addVolumes = false;
+ @Parameter(names = "--reset-security", description = "just update the security information")
+ boolean resetSecurity = false;
+ @Parameter(names = "--clear-instance-name", description = "delete any existing instance name without prompting")
+ boolean clearInstanceName = false;
+ @Parameter(names = "--instance-name", description = "the instance name, if not provided, will prompt")
+ String cliInstanceName;
+ @Parameter(names = "--password", description = "set the password on the command line")
+ String cliPassword;
+
+ byte[] rootpass = null;
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(Initialize.class.getName(), args);
+
+ try {
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ SecurityUtil.serverLogin(acuConf);
+ Configuration conf = CachedConfiguration.getInstance();
+
+ VolumeManager fs = VolumeManagerImpl.get(acuConf);
+
+ if (opts.resetSecurity) {
+ if (isInitialized(fs)) {
+ opts.rootpass = getRootPassword(opts);
+ initSecurity(opts, HdfsZooInstance.getInstance().getInstanceID());
+ } else {
+ log.fatal("Attempted to reset security on accumulo before it was initialized");
+ }
+ }
+
+ if (opts.addVolumes) {
+ addVolumes(fs);
+ }
+
+ if (!opts.resetSecurity && !opts.addVolumes)
+ if (!doInit(opts, conf, fs))
+ System.exit(-1);
+ } catch (Exception e) {
+ log.fatal(e, e);
+ throw new RuntimeException(e);
+ }
+ }
+}
[20/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 103cfe8,0000000..547ccc7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@@ -1,1601 -1,0 +1,1603 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceExistsException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.admin.DiskUsage;
+import org.apache.accumulo.core.client.admin.FindMax;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
+import org.apache.accumulo.core.client.impl.thrift.TDiskUsage;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.FateOperation;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.metadata.MetadataServicer;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public class TableOperationsImpl extends TableOperationsHelper {
+ private Instance instance;
+ private Credentials credentials;
+
+ public static final String CLONE_EXCLUDE_PREFIX = "!";
+
+ private static final Logger log = Logger.getLogger(TableOperations.class);
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the username/password for this connection
+ */
+ public TableOperationsImpl(Instance instance, Credentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ @Override
+ public SortedSet<String> list() {
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
+ TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
+ opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
+ return tableNames;
+ }
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ @Override
+ public boolean exists(String tableName) {
+ ArgumentChecker.notNull(tableName);
+ if (tableName.equals(MetadataTable.NAME) || tableName.equals(RootTable.NAME))
+ return true;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + " exists...");
+ boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
+ opTimer.stop("Checked existance of " + exists + " in %DURATION%");
+ return exists;
+ }
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ @Override
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, true, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, limitVersion, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ ArgumentChecker.notNull(tableName, timeType);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(UTF_8)));
+
+ Map<String,String> opts;
+ if (limitVersion)
+ opts = IteratorUtil.generateInitialTableProperties(limitVersion);
+ else
+ opts = Collections.emptyMap();
+
+ try {
+ doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_CREATE, args, opts);
+ } catch (TableNotFoundException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ private long beginFateOperation() throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.beginFateOperation(Tracer.traceInfo(), credentials.toThrift(instance));
+ } catch (TTransportException tte) {
+ log.debug("Failed to call beginFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ // This method is for retrying in the case of network failures; anything else it passes to the caller to deal with
+ private void executeFateOperation(long opid, FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
+ throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.executeFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid, op, args, opts, autoCleanUp);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call executeFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String waitForFateOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.waitForFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call waitForFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void finishFateOperation(long opid) throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.finishFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call finishFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
+ TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
+ return doFateOperation(op, args, opts, true);
+ }
+
+ String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
+ Long opid = null;
+
+ try {
+ opid = beginFateOperation();
+ executeFateOperation(opid, op, args, opts, !wait);
+ if (!wait) {
+ opid = null;
+ return null;
+ }
+ String ret = waitForFateOperation(opid);
+ return ret;
+ } catch (ThriftSecurityException e) {
+ String tableName = ByteBufferUtil.toString(args.get(0));
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(null, tableName, "Target table does not exist");
+ case NAMESPACE_DOESNT_EXIST:
+ throw new NamespaceNotFoundException(null, tableName, "Target namespace does not exist");
+ default:
+ String tableInfo = Tables.getPrintableTableInfoFromName(instance, tableName);
+ throw new AccumuloSecurityException(e.user, e.code, tableInfo, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case EXISTS:
+ throw new TableExistsException(e);
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_EXISTS:
+ throw new NamespaceExistsException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new NamespaceNotFoundException(e);
+ case OFFLINE:
+ throw new TableOfflineException(instance, null);
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e.getMessage(), e);
+ } finally {
+ Tables.clearCache(instance);
+ // always finish table op, even when exception
+ if (opid != null)
+ try {
+ finishFateOperation(opid);
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class SplitEnv {
+ private String tableName;
+ private String tableId;
+ private ExecutorService executor;
+ private CountDownLatch latch;
+ private AtomicReference<Exception> exception;
+
+ SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
+ this.tableName = tableName;
+ this.tableId = tableId;
+ this.executor = executor;
+ this.latch = latch;
+ this.exception = exception;
+ }
+ }
+
+ private class SplitTask implements Runnable {
+
+ private List<Text> splits;
+ private SplitEnv env;
+
+ SplitTask(SplitEnv env, List<Text> splits) {
+ this.env = env;
+ this.splits = splits;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (env.exception.get() != null)
+ return;
+
+ if (splits.size() <= 2) {
+ addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
+ for (int i = 0; i < splits.size(); i++)
+ env.latch.countDown();
+ return;
+ }
+
+ int mid = splits.size() / 2;
+
+ // split the middle split point to ensure that child task split different tablets and can therefore
+ // run in parallel
+ addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
+ env.latch.countDown();
+
+ env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+ env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
+
+ } catch (Exception e) {
+ env.exception.compareAndSet(null, e);
+ }
+ }
+
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<Text> splits = new ArrayList<Text>(partitionKeys);
+ // should be sorted because we copied from a sorted set, but that makes assumptions about
+ // how the copy was done so resort to be sure.
+ Collections.sort(splits);
+
+ CountDownLatch latch = new CountDownLatch(splits.size());
+ AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
+
+ ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
+ try {
+ executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
+
+ while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+ if (exception.get() != null) {
+ executor.shutdownNow();
+ Exception excep = exception.get();
+ if (excep instanceof TableNotFoundException)
+ throw (TableNotFoundException) excep;
+ else if (excep instanceof AccumuloException)
+ throw (AccumuloException) excep;
+ else if (excep instanceof AccumuloSecurityException)
+ throw (AccumuloSecurityException) excep;
+ else if (excep instanceof RuntimeException)
+ throw (RuntimeException) excep;
+ else
+ throw new RuntimeException(excep);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloServerException {
+ TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(tableId));
+
+ for (Text split : partitionKeys) {
+ boolean successful = false;
+ int attempt = 0;
+ long locationFailures = 0;
+
+ while (!successful) {
+
+ if (attempt > 0)
+ UtilWaitThread.sleep(100);
+
+ attempt++;
+
+ TabletLocation tl = tabLocator.locateTablet(credentials, split, false, false);
+
+ if (tl == null) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ continue;
+ }
+
+ try {
+ TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, ServerConfigurationUtil.getConfiguration(instance));
+ try {
+ OpTimer opTimer = null;
+ if (log.isTraceEnabled())
+ opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
+
+ client.splitTablet(Tracer.traceInfo(), credentials.toThrift(instance), tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
+
+ // just split it, might as well invalidate it in the cache
+ tabLocator.invalidateCache(tl.tablet_extent);
+
+ if (opTimer != null)
+ opTimer.stop("Split tablet in %DURATION%");
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+
+ } catch (TApplicationException tae) {
+ throw new AccumuloServerException(tl.tablet_location, tae);
+ } catch (TTransportException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ } catch (ThriftSecurityException e) {
+ Tables.clearCache(instance);
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (NotServingTabletException e) {
+ // Do not silently spin when we repeatedly fail to get the location for a tablet
+ locationFailures++;
+ if (5 == locationFailures || 0 == locationFailures % 50) {
+ log.warn("Having difficulty locating hosting tabletserver for split " + split + " on table " + tableName + ". Seen " + locationFailures
+ + " failures.");
+ }
+
+ tabLocator.invalidateCache(tl.tablet_extent);
+ continue;
+ } catch (TException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ }
+
+ successful = true;
+ }
+ }
+ }
+
+ @Override
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ end == null ? EMPTY : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_MERGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ end == null ? EMPTY : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE_RANGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
+
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ TreeMap<KeyExtent,String> tabletLocations = new TreeMap<KeyExtent,String>();
+
+ while (true) {
+ try {
+ tabletLocations.clear();
+ // the following method throws AccumuloException for some conditions that should be retried
+ MetadataServicer.forTableId(instance, credentials, tableId).getTabletLocations(tabletLocations);
+ break;
+ } catch (AccumuloSecurityException ase) {
+ throw ase;
+ } catch (Exception e) {
+ if (!Tables.exists(instance, tableId)) {
+ throw new TableNotFoundException(tableId, tableName, null);
+ }
+
+ if (e instanceof RuntimeException && e.getCause() instanceof AccumuloSecurityException) {
+ throw (AccumuloSecurityException) e.getCause();
+ }
+
+ log.info(e.getMessage() + " ... retrying ...");
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ ArrayList<Text> endRows = new ArrayList<Text>(tabletLocations.size());
+
+ for (KeyExtent ke : tabletLocations.keySet())
+ if (ke.getEndRow() != null)
+ endRows.add(ke.getEndRow());
+
+ return endRows;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
+ try {
+ return listSplits(tableName);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
+ Collection<Text> endRows = listSplits(tableName);
+
+ if (endRows.size() <= maxSplits)
+ return endRows;
+
+ double r = (maxSplits + 1) / (double) (endRows.size());
+ double pos = 0;
+
+ ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
+
+ int j = 0;
+ for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
+ pos += r;
+ while (pos > 1) {
+ subset.add(((ArrayList<Text>) endRows).get(i));
+ j++;
+ pos -= 1;
+ }
+ }
+
+ return subset;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
+ try {
+ return listSplits(tableName, maxSplits);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ }
+
+ @Override
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
+
+ ArgumentChecker.notNull(srcTableName, newTableName);
+
+ String srcTableId = Tables.getTableId(instance, srcTableName);
+
+ if (flush)
+ _flush(srcTableId, null, null, true);
+
+ if (propertiesToExclude == null)
+ propertiesToExclude = Collections.emptySet();
+
+ if (propertiesToSet == null)
+ propertiesToSet = Collections.emptyMap();
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ for (Entry<String,String> entry : propertiesToSet.entrySet()) {
+ if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
+ throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
+ opts.put(entry.getKey(), entry.getValue());
+ }
+
+ for (String prop : propertiesToExclude) {
+ opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
+ }
+
+ doTableFateOperation(newTableName, AccumuloException.class, FateOperation.TABLE_CLONE, args, opts);
+ }
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ @Override
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException {
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ doTableFateOperation(oldTableName, TableNotFoundException.class, FateOperation.TABLE_RENAME, args, opts);
+ }
+
+ /**
+ * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
+ */
+ @Override
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ flush(tableName, null, null, false);
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Flush a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+ _flush(tableId, start, end, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException {
+ compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (flush)
+ _flush(tableId, start, end, true);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ end == null ? EMPTY : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doFateOperation(FateOperation.TABLE_COMPACT, args, opts, wait);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ } catch (NamespaceExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ } catch (NamespaceNotFoundException e) {
+ throw new TableNotFoundException(null, tableName, "Namespace not found", e);
+ }
+ }
+
+ @Override
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
+ String tableId = Tables.getTableId(instance, tableName);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_CANCEL_COMPACT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ }
+
+ private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ try {
+ long flushID;
+
+ // used to pass the table name. but the tableid associated with a table name could change between calls.
+ // so pass the tableid to both calls
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ flushID = client.initiateFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.waitForFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
+ wait ? Long.MAX_VALUE : 1);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ } catch (ThriftSecurityException e) {
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(tableId, null, e.getMessage(), e);
+ default:
+ log.debug("flush security exception on table id " + tableId);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Sets a property on a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property, value);
+ try {
+ MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property, value);
+ }
+ });
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Removes a property from a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property);
+ try {
+ MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property);
+ }
+ });
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Gets properties of a table
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties)
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getTableConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), tableName);
+ }
+ }).entrySet();
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+
+ }
+
+ /**
+ * Sets a tables locality groups. A tables locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ // ensure locality groups do not overlap
+ HashSet<Text> all = new HashSet<Text>();
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+
+ if (!Collections.disjoint(all, entry.getValue())) {
+ throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
+ }
+
+ all.addAll(entry.getValue());
+ }
+
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+ Set<Text> colFams = entry.getValue();
+ String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
+ }
+
+ try {
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
+ } catch (AccumuloException e) {
+ if (e.getCause() instanceof TableNotFoundException)
+ throw (TableNotFoundException) e.getCause();
+ throw e;
+ }
+
+ // remove anything extraneous
+ String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
+ for (Entry<String,String> entry : getProperties(tableName)) {
+ String property = entry.getKey();
+ if (property.startsWith(prefix)) {
+ // this property configures a locality group, find out which
+ // one:
+ String[] parts = property.split("\\.");
+ String group = parts[parts.length - 1];
+
+ if (!groups.containsKey(group)) {
+ removeProperty(tableName, property);
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
+ AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
+ Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
+
+ Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
+ for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
+
+ HashSet<Text> colFams = new HashSet<Text>();
+
+ for (ByteSequence bs : entry.getValue()) {
+ colFams.add(new Text(bs.toArray()));
+ }
+
+ groups2.put(entry.getKey(), colFams);
+ }
+
+ return groups2;
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException {
+ ArgumentChecker.notNull(tableName, range);
+ if (maxSplits < 1)
+ throw new IllegalArgumentException("maximum splits must be >= 1");
+ if (maxSplits == 1)
+ return Collections.singleton(range);
+
+ Random random = new Random();
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ String tableId = Tables.getTableId(instance, tableName);
+ TabletLocator tl = TabletLocator.getLocator(instance, new Text(tableId));
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(credentials, Collections.singletonList(range), binnedRanges).isEmpty()) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+
+ log.warn("Unable to locate bins for specified range. Retrying.");
+ // sleep randomly between 100 and 200ms
+ UtilWaitThread.sleep(100 + random.nextInt(100));
+ binnedRanges.clear();
+ tl.invalidateCache();
+ }
+
+ // group key extents to get <= maxSplits
+ LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
+ List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
+
+ for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
+ unmergedExtents.addAll(map.keySet());
+
+ // the sort method is efficient for linked list
+ Collections.sort(unmergedExtents);
+
+ while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
+ if (unmergedExtents.size() >= 2) {
+ KeyExtent first = unmergedExtents.removeFirst();
+ KeyExtent second = unmergedExtents.removeFirst();
+ first.setEndRow(second.getEndRow());
+ mergedExtents.add(first);
+ } else {
+ mergedExtents.addAll(unmergedExtents);
+ unmergedExtents.clear();
+ unmergedExtents.addAll(mergedExtents);
+ mergedExtents.clear();
+ }
+
+ }
+
+ mergedExtents.addAll(unmergedExtents);
+
+ Set<Range> ranges = new HashSet<Range>();
+ for (KeyExtent k : mergedExtents)
+ ranges.add(k.toDataRange().clip(range));
+
+ return ranges;
+ }
+
+ // TODO Remove deprecation warning surppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ private Path checkPath(String dir, String kind, String type) throws IOException, AccumuloException {
+ Path ret;
+ FileSystem fs = VolumeConfiguration.getVolume(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance)).getFileSystem();
+
+ if (dir.contains(":")) {
+ ret = new Path(dir);
+ } else {
+ ret = fs.makeQualified(new Path(dir));
+ }
+
+ if (!fs.exists(ret))
+ throw new AccumuloException(kind + " import " + type + " directory " + dir + " does not exist!");
+
+ if (!fs.getFileStatus(ret).isDir()) {
+ throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not a directory!");
+ }
+
+ if (type.equals("failure")) {
+ FileStatus[] listStatus = fs.listStatus(ret);
+ if (listStatus != null && listStatus.length != 0) {
+ throw new AccumuloException("Bulk import failure directory " + ret + " is not empty");
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName, dir, failureDir);
+ // check for table existance
+ Tables.getTableId(instance, tableName);
+
+ Path dirPath = checkPath(dir, "Bulk", "");
+ Path failPath = checkPath(failureDir, "Bulk", "failure");
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
- ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(dirPath.toString().getBytes(UTF_8)),
++ ByteBuffer.wrap(failPath.toString().getBytes(UTF_8)), ByteBuffer.wrap((setTime + "").getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_BULK_IMPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ private void waitForTableStateTransition(String tableId, TableState expectedState) throws AccumuloException, TableNotFoundException,
+ AccumuloSecurityException {
+
+ Text startRow = null;
+ Text lastRow = null;
+
+ while (true) {
+
+ if (Tables.getTableState(instance, tableId) != expectedState) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != expectedState) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ throw new AccumuloException("Unexpected table state " + tableId + " " + Tables.getTableState(instance, tableId) + " != " + expectedState);
+ }
+ }
+
+ Range range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ if (startRow == null || lastRow == null)
+ range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ else
+ range = new Range(startRow, lastRow);
+
+ String metaTable = MetadataTable.NAME;
+ if (tableId.equals(MetadataTable.ID))
+ metaTable = RootTable.NAME;
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(metaTable, Authorizations.EMPTY);
+ scanner = new IsolatedScanner(scanner);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.setRange(range);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ int total = 0;
+ int waitFor = 0;
+ int holes = 0;
+ Text continueRow = null;
+ MapCounter<String> serverCounts = new MapCounter<String>();
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+
+ total++;
+
+ KeyExtent extent = null;
+ String future = null;
+ String current = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME))
+ future = entry.getValue().toString();
+
+ if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME))
+ current = entry.getValue().toString();
+
+ if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key))
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ if ((expectedState == TableState.ONLINE && current == null) || (expectedState == TableState.OFFLINE && (future != null || current != null))) {
+ if (continueRow == null)
+ continueRow = extent.getMetadataEntry();
+ waitFor++;
+ lastRow = extent.getMetadataEntry();
+
+ if (current != null)
+ serverCounts.increment(current, 1);
+ if (future != null)
+ serverCounts.increment(future, 1);
+ }
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ holes++;
+ }
+
+ lastExtent = extent;
+ }
+
+ if (continueRow != null) {
+ startRow = continueRow;
+ }
+
+ if (holes > 0 || total == 0) {
+ startRow = null;
+ lastRow = null;
+ }
+
+ if (waitFor > 0 || holes > 0 || total == 0) {
+ long waitTime;
+ long maxPerServer = 0;
+ if (serverCounts.size() > 0) {
+ maxPerServer = Collections.max(serverCounts.values());
+ waitTime = maxPerServer * 10;
+ } else
+ waitTime = waitFor * 10;
+ waitTime = Math.max(100, waitTime);
+ waitTime = Math.min(5000, waitTime);
+ log.trace("Waiting for " + waitFor + "(" + maxPerServer + ") tablets, startRow = " + startRow + " lastRow = " + lastRow + ", holes=" + holes
+ + " sleeping:" + waitTime + "ms");
+ UtilWaitThread.sleep(waitTime);
+ } else {
+ break;
+ }
+
+ }
+ }
+
+ @Override
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ offline(tableName, false);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ */
+ @Override
+ public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ String tableId = Tables.getTableId(instance, tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_OFFLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ if (wait)
+ waitForTableStateTransition(tableId, TableState.OFFLINE);
+ }
+
+ @Override
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ online(tableName, false);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ */
+ @Override
+ public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ String tableId = Tables.getTableId(instance, tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_ONLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ if (wait)
+ waitForTableStateTransition(tableId, TableState.ONLINE);
+ }
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ @Override
+ public void clearLocatorCache(String tableName) throws TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(Tables.getTableId(instance, tableName)));
+ tabLocator.invalidateCache();
+ }
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ @Override
+ public Map<String,String> tableIdMap() {
+ return Tables.getNameToIdMap(instance);
+ }
+
+ @Override
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, auths);
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, auths);
+ return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
+ }
+
+ @Override
+ public List<DiskUsage> getDiskUsage(Set<String> tableNames) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ List<TDiskUsage> diskUsages = null;
+ while (diskUsages == null) {
+ Pair<String,Client> pair = null;
+ try {
+ // this operation may us a lot of memory... its likely that connections to tabletservers hosting metadata tablets will be cached, so do not use cached
+ // connections
+ pair = ServerClient.getConnection(instance, false);
+ diskUsages = pair.getSecond().getDiskUsage(tableNames, credentials.toThrift(instance));
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new TableNotFoundException(e.getTableName(), new NamespaceNotFoundException(e));
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.getUser(), e.getCode());
+ } catch (TTransportException e) {
+ // some sort of communication error occurred, retry
+ if (pair == null) {
+ log.debug("Disk usage request failed. Pair is null. Retrying request...", e);
+ } else {
+ log.debug("Disk usage request failed " + pair.getFirst() + ", retrying ... ", e);
+ }
+ UtilWaitThread.sleep(100);
+ } catch (TException e) {
+ // may be a TApplicationException which indicates error on the server side
+ throw new AccumuloException(e);
+ } finally {
+ // must always return thrift connection
+ if (pair != null)
+ ServerClient.close(pair.getSecond());
+ }
+ }
+
+ List<DiskUsage> finalUsages = new ArrayList<DiskUsage>();
+ for (TDiskUsage diskUsage : diskUsages) {
+ finalUsages.add(new DiskUsage(new TreeSet<String>(diskUsage.getTables()), diskUsage.getUsage()));
+ }
+
+ return finalUsages;
+ }
+
+ public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
+ HashMap<String,String> props = new HashMap<String,String>();
+
+ ZipInputStream zis = new ZipInputStream(fs.open(path));
+ try {
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
- BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
+ try {
+ String line;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split("=", 2);
+ props.put(sa[0], sa[1]);
+ }
+ } finally {
+ in.close();
+ }
+
+ break;
+ }
+ }
+ } finally {
+ zis.close();
+ }
+ return props;
+ }
+
+ @Override
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, importDir);
+
+ try {
+ importDir = checkPath(importDir, "Table", "").toString();
+ } catch (IOException e) {
+ throw new AccumuloException(e);
+ }
+
+ try {
+ FileSystem fs = new Path(importDir).getFileSystem(CachedConfiguration.getInstance());
+ Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
+
+ for (Entry<String,String> entry : props.entrySet()) {
+ if (Property.isClassProperty(entry.getKey()) && !entry.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
+ Logger.getLogger(this.getClass()).info(
+ "Imported table sets '" + entry.getKey() + "' to '" + entry.getValue() + "'. Ensure this class is on Accumulo classpath.");
+ }
+ }
+
+ } catch (IOException ioe) {
+ Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
+ }
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(importDir.getBytes(UTF_8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_IMPORT, args, opts);
+ } catch (TableNotFoundException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ }
+
+ @Override
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, exportDir);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_EXPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, className, asTypeName);
+
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkTableClass(Tracer.traceInfo(), credentials.toThrift(instance), tableName, className, asTypeName);
+ }
+ });
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ @Override
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException {
+ testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
+ super.attachIterator(tableName, setting, scopes);
+ }
+
+ @Override
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ testClassLoad(tableName, constraintClassName, Constraint.class.getName());
+ return super.addConstraint(tableName, constraintClassName);
+ }
+
+ private void doTableFateOperation(String tableName, Class<? extends Exception> namespaceNotFoundExceptionClass, FateOperation op, List<ByteBuffer> args,
+ Map<String,String> opts) throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
+ try {
+ doFateOperation(op, args, opts);
+ } catch (NamespaceExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ } catch (NamespaceNotFoundException e) {
+ if (namespaceNotFoundExceptionClass == null) {
+ // should not happen
+ throw new AssertionError(e);
+ } else if (AccumuloException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
+ throw new AccumuloException("Cannot create table in non-existent namespace", e);
+ } else if (TableNotFoundException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
+ throw new TableNotFoundException(null, tableName, "Namespace not found", e);
+ } else {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
index 32b6e28,6414193..4de95c3
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
@@@ -16,8 -16,9 +16,10 @@@
*/
package org.apache.accumulo.core.client.impl;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.security.SecurityPermission;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
@@@ -118,75 -41,36 +120,75 @@@ public class Tables
if (sm != null) {
sm.checkPermission(TABLES_PERMISSION);
}
- return ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
}
-
+
private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
ZooCache zc = getZooCache(instance);
-
+
List<String> tableIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES);
-
TreeMap<String,String> tableMap = new TreeMap<String,String>();
-
+ Map<String,String> namespaceIdToNameMap = new HashMap<String,String>();
+
for (String tableId : tableIds) {
- byte[] tblPath = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
- if (tblPath != null) {
+ byte[] tableName = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
+ byte[] nId = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
+ String namespaceName = Namespaces.DEFAULT_NAMESPACE;
+ // create fully qualified table name
+ if (nId == null) {
+ namespaceName = null;
+ } else {
- String namespaceId = new String(nId, Constants.UTF8);
++ String namespaceId = new String(nId, UTF_8);
+ if (!namespaceId.equals(Namespaces.DEFAULT_NAMESPACE_ID)) {
+ try {
+ namespaceName = namespaceIdToNameMap.get(namespaceId);
+ if (namespaceName == null) {
+ namespaceName = Namespaces.getNamespaceName(instance, namespaceId);
+ namespaceIdToNameMap.put(namespaceId, namespaceName);
+ }
+ } catch (NamespaceNotFoundException e) {
+ log.error("Table (" + tableId + ") contains reference to namespace (" + namespaceId + ") that doesn't exist", e);
+ continue;
+ }
+ }
+ }
+ if (tableName != null && namespaceName != null) {
- String tableNameStr = qualified(new String(tableName, Constants.UTF8), namespaceName);
++ String tableNameStr = qualified(new String(tableName, UTF_8), namespaceName);
if (nameAsKey)
- tableMap.put(new String(tblPath, UTF_8), tableId);
+ tableMap.put(tableNameStr, tableId);
else
- tableMap.put(tableId, new String(tblPath, UTF_8));
+ tableMap.put(tableId, tableNameStr);
}
}
-
+
return tableMap;
}
-
+
public static String getTableId(Instance instance, String tableName) throws TableNotFoundException {
+ try {
+ return _getTableId(instance, tableName);
+ } catch (NamespaceNotFoundException e) {
+ throw new TableNotFoundException(tableName, e);
+ }
+ }
+
+ public static String _getTableId(Instance instance, String tableName) throws NamespaceNotFoundException, TableNotFoundException {
String tableId = getNameToIdMap(instance).get(tableName);
- if (tableId == null)
- throw new TableNotFoundException(tableId, tableName, null);
+ if (tableId == null) {
+ // maybe the table exist, but the cache was not updated yet... so try to clear the cache and check again
+ clearCache(instance);
+ tableId = getNameToIdMap(instance).get(tableName);
+ if (tableId == null) {
+ String namespace = qualify(tableName).getFirst();
+ if (Namespaces.getNameToIdMap(instance).containsKey(namespace))
+ throw new TableNotFoundException(null, tableName, null);
+ else
+ throw new NamespaceNotFoundException(null, namespace, null);
+ }
+ }
return tableId;
}
-
+
public static String getTableName(Instance instance, String tableId) throws TableNotFoundException {
String tableName = getIdToNameMap(instance).get(tableId);
if (tableName == null)
@@@ -250,65 -113,11 +252,65 @@@
byte[] state = zc.get(statePath);
if (state == null)
return TableState.UNKNOWN;
-
+
- return TableState.valueOf(new String(state, Constants.UTF8));
+ return TableState.valueOf(new String(state, UTF_8));
}
-
+
public static long getCacheResetCount() {
return cacheResetCount.get();
}
+
+ public static String qualified(String tableName) {
+ return qualified(tableName, Namespaces.DEFAULT_NAMESPACE);
+ }
+
+ public static String qualified(String tableName, String defaultNamespace) {
+ Pair<String,String> qualifiedTableName = Tables.qualify(tableName, defaultNamespace);
+ if (Namespaces.DEFAULT_NAMESPACE.equals(qualifiedTableName.getFirst()))
+ return qualifiedTableName.getSecond();
+ else
+ return qualifiedTableName.toString("", ".", "");
+ }
+
+ public static Pair<String,String> qualify(String tableName) {
+ return qualify(tableName, Namespaces.DEFAULT_NAMESPACE);
+ }
+
+ public static Pair<String,String> qualify(String tableName, String defaultNamespace) {
+ if (!tableName.matches(Tables.VALID_NAME_REGEX))
+ throw new IllegalArgumentException("Invalid table name '" + tableName + "'");
+ if (MetadataTable.OLD_NAME.equals(tableName))
+ tableName = MetadataTable.NAME;
+ if (tableName.contains(".")) {
+ String[] s = tableName.split("\\.", 2);
+ return new Pair<String,String>(s[0], s[1]);
+ }
+ return new Pair<String,String>(defaultNamespace, tableName);
+ }
+
+ /**
+ * Returns the namespace id for a given table ID.
+ *
+ * @param instance
+ * The Accumulo Instance
+ * @param tableId
+ * The tableId
+ * @return The namespace id which this table resides in.
+ * @throws IllegalArgumentException
+ * if the table doesn't exist in ZooKeeper
+ */
+ public static String getNamespaceId(Instance instance, String tableId) throws IllegalArgumentException {
+ ArgumentChecker.notNull(instance, tableId);
+
+ ZooCache zc = getZooCache(instance);
+ byte[] n = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
+
+ // We might get null out of ZooCache if this tableID doesn't exist
+ if (null == n) {
+ throw new IllegalArgumentException("Table with id " + tableId + " does not exist");
+ }
+
- return new String(n, Constants.UTF8);
++ return new String(n, UTF_8);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index d58d1e4,a9f5b61..de7bcb2
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@@ -81,10 -82,10 +82,10 @@@ public class Writer
throw new IllegalArgumentException("Can not add empty mutations");
while (true) {
- TabletLocation tabLoc = TabletLocator.getInstance(instance, table).locateTablet(new Text(m.getRow()), false, true, credentials);
+ TabletLocation tabLoc = TabletLocator.getLocator(instance, table).locateTablet(credentials, new Text(m.getRow()), false, true);
if (tabLoc == null) {
- log.trace("No tablet location found for row " + new String(m.getRow(), Constants.UTF8));
+ log.trace("No tablet location found for row " + new String(m.getRow(), UTF_8));
UtilWaitThread.sleep(500);
continue;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
index a071d07,0000000..bf365ef
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
@@@ -1,40 -1,0 +1,40 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.lexicoder;
+
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
+
+/**
+ * This lexicoder encodes/decodes a given String to/from bytes without further processing. It can be combined with other encoders like the
+ * {@link ReverseLexicoder} to flip the default sort order.
+ *
+ * @since 1.6.0
+ */
+
+public class StringLexicoder implements Lexicoder<String> {
+
+ @Override
+ public byte[] encode(String data) {
- return data.getBytes(Constants.UTF8);
++ return data.getBytes(UTF_8);
+ }
+
+ @Override
+ public String decode(byte[] data) {
- return new String(data, Constants.UTF8);
++ return new String(data, UTF_8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 500047c,23540f2..563610a
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -27,8 -27,7 +29,7 @@@ import java.util.HashSet
import java.util.List;
import java.util.Set;
- import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.ZooKeeperInstance;
@@@ -203,24 -189,14 +204,24 @@@ public class RangeInputSplit extends In
}
if (in.readBoolean()) {
- String tokenClass = in.readUTF();
- byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
- byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
-
- try {
- token = CredentialHelper.extractToken(tokenClass, tokenBytes);
- } catch (AccumuloSecurityException e) {
- throw new IOException(e);
+ int ordinal = in.readInt();
+ this.tokenSource = TokenSource.values()[ordinal];
+
+ switch (this.tokenSource) {
+ case INLINE:
+ String tokenClass = in.readUTF();
- byte[] base64TokenBytes = in.readUTF().getBytes(Constants.UTF8);
++ byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
+ byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
+
+ this.token = AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes);
+ break;
+
+ case FILE:
+ this.tokenFile = in.readUTF();
+
+ break;
+ default:
+ throw new IOException("Cannot parse unknown TokenSource ordinal");
}
}
[04/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 0e02f05,0000000..7158ea8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@@ -1,598 -1,0 +1,598 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.log;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_FINISH;
+import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_START;
+import static org.apache.accumulo.tserver.logger.LogEvents.DEFINE_TABLET;
+import static org.apache.accumulo.tserver.logger.LogEvents.MANY_MUTATIONS;
+import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.crypto.CryptoModule;
+import org.apache.accumulo.core.security.crypto.CryptoModuleFactory;
+import org.apache.accumulo.core.security.crypto.CryptoModuleParameters;
+import org.apache.accumulo.core.security.crypto.DefaultCryptoModule;
+import org.apache.accumulo.core.security.crypto.NoFlushOutputStream;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.tserver.TabletMutations;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Wrap a connection to a logger.
+ *
+ */
+public class DfsLogger {
+ public static final String LOG_FILE_HEADER_V2 = "--- Log File Header (v2) ---";
+ public static final String LOG_FILE_HEADER_V3 = "--- Log File Header (v3) ---";
+
+ private static Logger log = Logger.getLogger(DfsLogger.class);
+
+ public static class LogClosedException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public LogClosedException() {
+ super("LogClosed");
+ }
+ }
+
+ /**
+ * A well-timed tabletserver failure could result in an incomplete header written to a write-ahead log. This exception is thrown when the header cannot be
+ * read from a WAL which should only happen when the tserver dies as described.
+ */
+ public static class LogHeaderIncompleteException extends IOException {
+ private static final long serialVersionUID = 1l;
+
+ public LogHeaderIncompleteException(String msg) {
+ super(msg);
+ }
+
+ public LogHeaderIncompleteException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+ public LogHeaderIncompleteException(Throwable cause) {
+ super(cause);
+ }
+ }
+
+ public static class DFSLoggerInputStreams {
+
+ private FSDataInputStream originalInput;
+ private DataInputStream decryptingInputStream;
+
+ public DFSLoggerInputStreams(FSDataInputStream originalInput, DataInputStream decryptingInputStream) {
+ this.originalInput = originalInput;
+ this.decryptingInputStream = decryptingInputStream;
+ }
+
+ public FSDataInputStream getOriginalInput() {
+ return originalInput;
+ }
+
+ public void setOriginalInput(FSDataInputStream originalInput) {
+ this.originalInput = originalInput;
+ }
+
+ public DataInputStream getDecryptingInputStream() {
+ return decryptingInputStream;
+ }
+
+ public void setDecryptingInputStream(DataInputStream decryptingInputStream) {
+ this.decryptingInputStream = decryptingInputStream;
+ }
+ }
+
+ public interface ServerResources {
+ AccumuloConfiguration getConfiguration();
+
+ VolumeManager getFileSystem();
+
+ Set<TServerInstance> getCurrentTServers();
+ }
+
+ private final LinkedBlockingQueue<DfsLogger.LogWork> workQueue = new LinkedBlockingQueue<DfsLogger.LogWork>();
+
+ private final Object closeLock = new Object();
+
+ private static final DfsLogger.LogWork CLOSED_MARKER = new DfsLogger.LogWork(null);
+
+ private static final LogFileValue EMPTY = new LogFileValue();
+
+ private boolean closed = false;
+
+ private class LogSyncingTask implements Runnable {
+
+ @Override
+ public void run() {
+ ArrayList<DfsLogger.LogWork> work = new ArrayList<DfsLogger.LogWork>();
+ boolean sawClosedMarker = false;
+ while (!sawClosedMarker) {
+ work.clear();
+
+ try {
+ work.add(workQueue.take());
+ } catch (InterruptedException ex) {
+ continue;
+ }
+ workQueue.drainTo(work);
+
+ try {
+ sync.invoke(logFile);
+ } catch (Exception ex) {
+ log.warn("Exception syncing " + ex);
+ for (DfsLogger.LogWork logWork : work) {
+ logWork.exception = ex;
+ }
+ }
+
+ for (DfsLogger.LogWork logWork : work)
+ if (logWork == CLOSED_MARKER)
+ sawClosedMarker = true;
+ else
+ logWork.latch.countDown();
+ }
+ }
+ }
+
+ static class LogWork {
+ CountDownLatch latch;
+ volatile Exception exception;
+
+ public LogWork(CountDownLatch latch) {
+ this.latch = latch;
+ }
+ }
+
+ public static class LoggerOperation {
+ private final LogWork work;
+
+ public LoggerOperation(LogWork work) {
+ this.work = work;
+ }
+
+ public void await() throws IOException {
+ try {
+ work.latch.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ if (work.exception != null) {
+ if (work.exception instanceof IOException)
+ throw (IOException) work.exception;
+ else if (work.exception instanceof RuntimeException)
+ throw (RuntimeException) work.exception;
+ else
+ throw new RuntimeException(work.exception);
+ }
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // filename is unique
+ if (obj == null)
+ return false;
+ if (obj instanceof DfsLogger)
+ return getFileName().equals(((DfsLogger) obj).getFileName());
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ // filename is unique
+ return getFileName().hashCode();
+ }
+
+ private final ServerResources conf;
+ private FSDataOutputStream logFile;
+ private DataOutputStream encryptingLogFile = null;
+ private Method sync;
+ private String logPath;
+ private Daemon syncThread;
+
+ /* Track what's actually in +r/!0 for this logger ref */
+ private String metaReference;
+
+ public DfsLogger(ServerResources conf) throws IOException {
+ this.conf = conf;
+ }
+
+ /**
+ * Refernce a pre-existing log file.
+ *
+ * @param meta
+ * the cq for the "log" entry in +r/!0
+ */
+ public DfsLogger(ServerResources conf, String filename, String meta) throws IOException {
+ this.conf = conf;
+ this.logPath = filename;
+ metaReference = meta;
+ }
+
+ public static DFSLoggerInputStreams readHeaderAndReturnStream(VolumeManager fs, Path path, AccumuloConfiguration conf) throws IOException {
+ FSDataInputStream input = fs.open(path);
+ DataInputStream decryptingInput = null;
+
+ byte[] magic = DfsLogger.LOG_FILE_HEADER_V3.getBytes();
+ byte[] magicBuffer = new byte[magic.length];
+ try {
+ input.readFully(magicBuffer);
+ if (Arrays.equals(magicBuffer, magic)) {
+ // additional parameters it needs from the underlying stream.
+ String cryptoModuleClassname = input.readUTF();
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoModuleClassname);
+
+ // Create the parameters and set the input stream into those parameters
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setEncryptedInputStream(input);
+
+ // Create the plaintext input stream from the encrypted one
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ if (params.getPlaintextInputStream() instanceof DataInputStream) {
+ decryptingInput = (DataInputStream) params.getPlaintextInputStream();
+ } else {
+ decryptingInput = new DataInputStream(params.getPlaintextInputStream());
+ }
+ } else {
+ input.seek(0);
+ byte[] magicV2 = DfsLogger.LOG_FILE_HEADER_V2.getBytes();
+ byte[] magicBufferV2 = new byte[magicV2.length];
+ input.readFully(magicBufferV2);
+
+ if (Arrays.equals(magicBufferV2, magicV2)) {
+ // Log files from 1.5 dump their options in raw to the logger files. Since we don't know the class
+ // that needs to read those files, we can make a couple of basic assumptions. Either it's going to be
+ // the NullCryptoModule (no crypto) or the DefaultCryptoModule.
+
+ // If it's null, we won't have any parameters whatsoever. First, let's attempt to read
+ // parameters
+ Map<String,String> opts = new HashMap<String,String>();
+ int count = input.readInt();
+ for (int i = 0; i < count; i++) {
+ String key = input.readUTF();
+ String value = input.readUTF();
+ opts.put(key, value);
+ }
+
+ if (opts.size() == 0) {
+ // NullCryptoModule, we're done
+ decryptingInput = input;
+ } else {
+
+ // The DefaultCryptoModule will want to read the parameters from the underlying file, so we will put the file back to that spot.
+ org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory
+ .getCryptoModule(DefaultCryptoModule.class.getName());
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ input.seek(0);
+ input.readFully(magicBufferV2);
+ params.setEncryptedInputStream(input);
+
+ params = cryptoModule.getDecryptingInputStream(params);
+ if (params.getPlaintextInputStream() instanceof DataInputStream) {
+ decryptingInput = (DataInputStream) params.getPlaintextInputStream();
+ } else {
+ decryptingInput = new DataInputStream(params.getPlaintextInputStream());
+ }
+ }
+
+ } else {
+
+ input.seek(0);
+ decryptingInput = input;
+ }
+
+ }
+ } catch (EOFException e) {
+ log.warn("Got EOFException trying to read WAL header information, assuming the rest of the file (" + path + ") has no data.");
+ // A TabletServer might have died before the (complete) header was written
+ throw new LogHeaderIncompleteException(e);
+ }
+
+ return new DFSLoggerInputStreams(input, decryptingInput);
+ }
+
+ public synchronized void open(String address) throws IOException {
+ String filename = UUID.randomUUID().toString();
+ String logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
+
+ log.debug("DfsLogger.open() begin");
+ VolumeManager fs = conf.getFileSystem();
+
+ logPath = fs.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
+
+ metaReference = toString();
+ try {
+ short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
+ if (replication == 0)
+ replication = fs.getDefaultReplication(new Path(logPath));
+ long blockSize = conf.getConfiguration().getMemoryInBytes(Property.TSERV_WAL_BLOCKSIZE);
+ if (blockSize == 0)
+ blockSize = (long) (conf.getConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1);
+ if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
+ logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
+ else
+ logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
+
+ String syncMethod = conf.getConfiguration().get(Property.TSERV_WAL_SYNC_METHOD);
+ try {
+ // hsync: send data to datanodes and sync the data to disk
+ sync = logFile.getClass().getMethod(syncMethod);
+ } catch (Exception ex) {
+ log.warn("Could not find configured " + syncMethod + " method, trying to fall back to old Hadoop sync method", ex);
+
+ try {
+ // sync: send data to datanodes
+ sync = logFile.getClass().getMethod("sync");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // Initialize the crypto operations.
+ org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf
+ .getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
+
+ // Initialize the log file with a header and the crypto params used to set up this log file.
- logFile.write(LOG_FILE_HEADER_V3.getBytes(Constants.UTF8));
++ logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
+
+ NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
+ params.setPlaintextOutputStream(nfos);
+
+ // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
+ // so that that crypto module can re-read its own parameters.
+
+ logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
+
+ params = cryptoModule.getEncryptingOutputStream(params);
+ OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
+
+ // If the module just kicks back our original stream, then just use it, don't wrap it in
+ // another data OutputStream.
+ if (encipheringOutputStream == nfos) {
+ log.debug("No enciphering, using raw output stream");
+ encryptingLogFile = nfos;
+ } else {
+ log.debug("Enciphering found, wrapping in DataOutputStream");
+ encryptingLogFile = new DataOutputStream(encipheringOutputStream);
+ }
+
+ LogFileKey key = new LogFileKey();
+ key.event = OPEN;
+ key.tserverSession = filename;
+ key.filename = filename;
+ write(key, EMPTY);
+ sync.invoke(logFile);
+ log.debug("Got new write-ahead log: " + this);
+ } catch (Exception ex) {
+ if (logFile != null)
+ logFile.close();
+ logFile = null;
+ encryptingLogFile = null;
+ throw new IOException(ex);
+ }
+
+ syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
+ syncThread.setName("Accumulo WALog thread " + toString());
+ syncThread.start();
+ }
+
+ @Override
+ public String toString() {
+ String fileName = getFileName();
+ if (fileName.contains(":"))
+ return getLogger() + "/" + getFileName();
+ return fileName;
+ }
+
+ /**
+ * get the cq needed to reference this logger's entry in +r/!0
+ */
+ public String getMeta() {
+ if (null == metaReference) {
+ throw new IllegalStateException("logger doesn't have meta reference. " + this);
+ }
+ return metaReference;
+ }
+
+ public String getFileName() {
+ return logPath.toString();
+ }
+
+ public void close() throws IOException {
+
+ synchronized (closeLock) {
+ if (closed)
+ return;
+ // after closed is set to true, nothing else should be added to the queue
+ // CLOSED_MARKER should be the last thing on the queue, therefore when the
+ // background thread sees the marker and exits there should be nothing else
+ // to process... so nothing should be left waiting for the background
+ // thread to do work
+ closed = true;
+ workQueue.add(CLOSED_MARKER);
+ }
+
+ // wait for background thread to finish before closing log file
+ if (syncThread != null) {
+ try {
+ syncThread.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // expect workq should be empty at this point
+ if (workQueue.size() != 0) {
+ log.error("WAL work queue not empty after sync thread exited");
+ throw new IllegalStateException("WAL work queue not empty after sync thread exited");
+ }
+
+ if (encryptingLogFile != null)
+ try {
+ logFile.close();
+ } catch (IOException ex) {
+ log.error(ex);
+ throw new LogClosedException();
+ }
+ }
+
+ public synchronized void defineTablet(int seq, int tid, KeyExtent tablet) throws IOException {
+ // write this log to the METADATA table
+ final LogFileKey key = new LogFileKey();
+ key.event = DEFINE_TABLET;
+ key.seq = seq;
+ key.tid = tid;
+ key.tablet = tablet;
+ try {
+ write(key, EMPTY);
+ sync.invoke(logFile);
+ } catch (IllegalArgumentException e) {
+ log.error("Signature of sync method changed. Accumulo is likely incompatible with this version of Hadoop.");
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ log.error("Could not invoke sync method due to permission error.");
+ throw new RuntimeException(e);
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException) {
+ throw (IOException) cause;
+ } else if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ } else if (cause instanceof Error) {
+ throw (Error) cause;
+ } else {
+ // Cause is null, or some other checked exception that was added later.
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private synchronized void write(LogFileKey key, LogFileValue value) throws IOException {
+ key.write(encryptingLogFile);
+ value.write(encryptingLogFile);
+ encryptingLogFile.flush();
+ }
+
+ public LoggerOperation log(int seq, int tid, Mutation mutation) throws IOException {
+ return logManyTablets(Collections.singletonList(new TabletMutations(tid, seq, Collections.singletonList(mutation))));
+ }
+
+ private LoggerOperation logFileData(List<Pair<LogFileKey,LogFileValue>> keys) throws IOException {
+ DfsLogger.LogWork work = new DfsLogger.LogWork(new CountDownLatch(1));
+ synchronized (DfsLogger.this) {
+ try {
+ for (Pair<LogFileKey,LogFileValue> pair : keys) {
+ write(pair.getFirst(), pair.getSecond());
+ }
+ } catch (ClosedChannelException ex) {
+ throw new LogClosedException();
+ } catch (Exception e) {
+ log.error(e, e);
+ work.exception = e;
+ }
+ }
+
+ synchronized (closeLock) {
+ // use a different lock for close check so that adding to work queue does not need
+ // to wait on walog I/O operations
+
+ if (closed)
+ throw new LogClosedException();
+ workQueue.add(work);
+ }
+
+ return new LoggerOperation(work);
+ }
+
+ public LoggerOperation logManyTablets(List<TabletMutations> mutations) throws IOException {
+ List<Pair<LogFileKey,LogFileValue>> data = new ArrayList<Pair<LogFileKey,LogFileValue>>();
+ for (TabletMutations tabletMutations : mutations) {
+ LogFileKey key = new LogFileKey();
+ key.event = MANY_MUTATIONS;
+ key.seq = tabletMutations.getSeq();
+ key.tid = tabletMutations.getTid();
+ LogFileValue value = new LogFileValue();
+ value.mutations = tabletMutations.getMutations();
+ data.add(new Pair<LogFileKey,LogFileValue>(key, value));
+ }
+ return logFileData(data);
+ }
+
+ public LoggerOperation minorCompactionFinished(int seq, int tid, String fqfn) throws IOException {
+ LogFileKey key = new LogFileKey();
+ key.event = COMPACTION_FINISH;
+ key.seq = seq;
+ key.tid = tid;
+ return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
+ }
+
+ public LoggerOperation minorCompactionStarted(int seq, int tid, String fqfn) throws IOException {
+ LogFileKey key = new LogFileKey();
+ key.event = COMPACTION_START;
+ key.seq = seq;
+ key.tid = tid;
+ key.filename = fqfn;
+ return logFileData(Collections.singletonList(new Pair<LogFileKey,LogFileValue>(key, EMPTY)));
+ }
+
+ public String getLogger() {
+ String parts[] = logPath.split("/");
+ return StringUtil.join(Arrays.asList(parts[parts.length - 2].split("[+]")), ":");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
index 997f71b,0000000..2c807fd
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
@@@ -1,96 -1,0 +1,97 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.logger;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.hadoop.io.Writable;
+
+public class LogFileValue implements Writable {
+
+ private static final List<Mutation> empty = Collections.emptyList();
+
+ public List<Mutation> mutations = empty;
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int count = in.readInt();
+ mutations = new ArrayList<Mutation>(count);
+ for (int i = 0; i < count; i++) {
+ ServerMutation mutation = new ServerMutation();
+ mutation.readFields(in);
+ mutations.add(mutation);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(mutations.size());
+ for (Mutation m : mutations) {
+ m.write(out);
+ }
+ }
+
+ public static void print(LogFileValue value) {
+ System.out.println(value.toString());
+ }
+
+ private static String displayLabels(byte[] labels) {
- String s = new String(labels, Constants.UTF8);
++ String s = new String(labels, UTF_8);
+ s = s.replace("&", " & ");
+ s = s.replace("|", " | ");
+ return s;
+ }
+
+ public static String format(LogFileValue lfv, int maxMutations) {
+ if (lfv.mutations.size() == 0)
+ return "";
+ StringBuilder builder = new StringBuilder();
+ builder.append(lfv.mutations.size() + " mutations:\n");
+ int i = 0;
+ for (Mutation m : lfv.mutations) {
+ if (i++ >= maxMutations) {
+ builder.append("...");
+ break;
+ }
- builder.append(" ").append(new String(m.getRow(), Constants.UTF8)).append("\n");
++ builder.append(" ").append(new String(m.getRow(), UTF_8)).append("\n");
+ for (ColumnUpdate update : m.getUpdates()) {
+ String value = new String(update.getValue());
- builder.append(" ").append(new String(update.getColumnFamily(), Constants.UTF8)).append(":")
- .append(new String(update.getColumnQualifier(), Constants.UTF8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
++ builder.append(" ").append(new String(update.getColumnFamily(), UTF_8)).append(":")
++ .append(new String(update.getColumnQualifier(), UTF_8)).append(" ").append(update.hasTimestamp() ? "[user]:" : "[system]:")
+ .append(update.getTimestamp()).append(" [").append(displayLabels(update.getColumnVisibility())).append("] ")
+ .append(update.isDeleted() ? "<deleted>" : value).append("\n");
+ }
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public String toString() {
+ return format(this, 5);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index c3f4fd0,0000000..29eefc8
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -1,178 -1,0 +1,179 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.logger;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
+import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException;
+import org.apache.accumulo.tserver.log.MultiReader;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+public class LogReader {
+ private static final Logger log = Logger.getLogger(LogReader.class);
+
+ static class Opts extends Help {
+ @Parameter(names = "-r", description = "print only mutations associated with the given row")
+ String row;
+ @Parameter(names = "-m", description = "limit the number of mutations printed per row")
+ int maxMutations = 5;
+ @Parameter(names = "-t", description = "print only mutations that fall within the given key extent")
+ String extent;
+ @Parameter(names = "-p", description = "search for a row that matches the given regex")
+ String regexp;
+ @Parameter(description = "<logfile> { <logfile> ... }")
+ List<String> files = new ArrayList<String>();
+ }
+
+ /**
+ * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
+ *
+ * @param args
+ * - first argument is the file to print
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(LogReader.class.getName(), args);
+ VolumeManager fs = VolumeManagerImpl.get();
+
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
+ Text row = null;
+ if (opts.files.isEmpty()) {
+ new JCommander(opts).usage();
+ return;
+ }
+ if (opts.row != null)
+ row = new Text(opts.row);
+ if (opts.extent != null) {
+ String sa[] = opts.extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (opts.regexp != null) {
+ Pattern pattern = Pattern.compile(opts.regexp);
+ rowMatcher = pattern.matcher("");
+ }
+
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
+ for (String file : opts.files) {
+
+ Path path = new Path(file);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ if (fs.isFile(path)) {
+ // read log entries from a simple hdfs file
+ DFSLoggerInputStreams streams;
+ try {
+ streams = DfsLogger.readHeaderAndReturnStream(fs, path, ServerConfiguration.getSiteConfiguration());
+ } catch (LogHeaderIncompleteException e) {
+ log.warn("Could not read header for " + path + ". Ignoring...");
+ continue;
+ }
+ DataInputStream input = streams.getDecryptingInputStream();
+
+ try {
+ while (true) {
+ try {
+ key.readFields(input);
+ value.readFields(input);
+ } catch (EOFException ex) {
+ break;
+ }
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ } finally {
+ input.close();
+ }
+ } else {
+ // read the log entries sorted in a map file
+ MultiReader input = new MultiReader(fs, path);
+ while (input.next(key, value)) {
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
+ }
+ }
+ }
+ }
+
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
+ if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
+ boolean found = false;
+ for (Mutation m : value.mutations) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
+ found = true;
+ break;
+ }
+
+ if (rowMatcher != null) {
- rowMatcher.reset(new String(m.getRow(), Constants.UTF8));
++ rowMatcher.reset(new String(m.getRow(), UTF_8));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return;
+ } else {
+ return;
+ }
+
+ }
+
+ System.out.println(key);
+ System.out.println(LogFileValue.format(value, maxMutations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapConcurrencyTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/NativeMapStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
index a679c48,ef6311b..3c20858
--- a/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestBinaryRows.java
@@@ -71,98 -72,71 +72,98 @@@ public class TestBinaryRows
return l;
}
- static class Opts extends ClientOnRequiredTable {
+ public static class Opts extends ClientOnRequiredTable {
@Parameter(names="--mode", description="either 'ingest', 'delete', 'randomLookups', 'split', 'verify', 'verifyDeleted'", required=true)
- String mode;
+ public String mode;
@Parameter(names="--start", description="the lowest numbered row")
- long start = 0;
+ public long start = 0;
@Parameter(names="--count", description="number of rows to ingest", required=true)
- long num = 0;
+ public long num = 0;
}
- public static void main(String[] args) {
- Opts opts = new Opts();
- BatchWriterOpts bwOpts = new BatchWriterOpts();
- ScannerOpts scanOpts = new ScannerOpts();
- opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
+ public static void runTest(Connector connector, Opts opts, BatchWriterOpts bwOpts, ScannerOpts scanOpts) throws Exception {
- try {
- Connector connector = opts.getConnector();
-
- final Text CF = new Text("cf"), CQ = new Text("cq");
- final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
+ final Text CF = new Text("cf"), CQ = new Text("cq");
- final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
++ final byte[] CF_BYTES = "cf".getBytes(UTF_8), CQ_BYTES = "cq".getBytes(UTF_8);
+ if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
+ boolean delete = opts.mode.equals("delete");
- if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
- BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
- boolean delete = opts.mode.equals("delete");
-
- for (long i = 0; i < opts.num; i++) {
- byte[] row = encodeLong(i + opts.start);
- String value = "" + (i + opts.start);
-
- Mutation m = new Mutation(new Text(row));
- if (delete) {
- m.putDelete(CF, CQ);
- } else {
- m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
- }
- bw.addMutation(m);
+ for (long i = 0; i < opts.num; i++) {
+ byte[] row = encodeLong(i + opts.start);
+ String value = "" + (i + opts.start);
+
+ Mutation m = new Mutation(new Text(row));
+ if (delete) {
+ m.putDelete(CF, CQ);
+ } else {
- m.put(CF, CQ, new Value(value.getBytes(Constants.UTF8)));
++ m.put(CF, CQ, new Value(value.getBytes(UTF_8)));
}
-
- bw.close();
- } else if (opts.mode.equals("verifyDeleted")) {
- Scanner s = connector.createScanner(opts.tableName, opts.auths);
- s.setBatchSize(scanOpts.scanBatchSize);
- Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
- Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
- s.setBatchSize(50000);
- s.setRange(new Range(startKey, stopKey));
-
- for (Entry<Key,Value> entry : s) {
- System.err.println("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
- System.err.println("exiting...");
- System.exit(1);
- }
-
- } else if (opts.mode.equals("verify")) {
- long t1 = System.currentTimeMillis();
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ } else if (opts.mode.equals("verifyDeleted")) {
+ Scanner s = connector.createScanner(opts.tableName, opts.auths);
+ s.setBatchSize(scanOpts.scanBatchSize);
+ Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
+ Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
+ s.setBatchSize(50000);
+ s.setRange(new Range(startKey, stopKey));
+
+ for (Entry<Key,Value> entry : s) {
+ throw new Exception("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
+ }
+
+ } else if (opts.mode.equals("verify")) {
+ long t1 = System.currentTimeMillis();
+
+ Scanner s = connector.createScanner(opts.tableName, opts.auths);
+ Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
+ Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
+ s.setBatchSize(scanOpts.scanBatchSize);
+ s.setRange(new Range(startKey, stopKey));
+
+ long i = opts.start;
+
+ for (Entry<Key,Value> e : s) {
+ Key k = e.getKey();
+ Value v = e.getValue();
+
+ checkKeyValue(i, k, v);
+
+ i++;
+ }
+
+ if (i != opts.start + opts.num) {
+ throw new Exception("ERROR : did not see expected number of rows, saw " + (i - opts.start) + " expected " + opts.num);
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ System.out.printf("time : %9.2f secs%n", ((t2 - t1) / 1000.0));
+ System.out.printf("rate : %9.2f entries/sec%n", opts.num / ((t2 - t1) / 1000.0));
+
+ } else if (opts.mode.equals("randomLookups")) {
+ int numLookups = 1000;
+
+ Random r = new Random();
+
+ long t1 = System.currentTimeMillis();
+
+ for (int i = 0; i < numLookups; i++) {
+ long row = ((r.nextLong() & 0x7fffffffffffffffl) % opts.num) + opts.start;
Scanner s = connector.createScanner(opts.tableName, opts.auths);
- Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
- Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
s.setBatchSize(scanOpts.scanBatchSize);
+ Key startKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
+ Key stopKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], 0);
s.setRange(new Range(startKey, stopKey));
- long i = opts.start;
+ Iterator<Entry<Key,Value>> si = s.iterator();
- for (Entry<Key,Value> e : s) {
+ if (si.hasNext()) {
+ Entry<Key,Value> e = si.next();
Key k = e.getKey();
Value v = e.getValue();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestIngest.java
index bc3ff05,5d8e268..c3e77af
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@@ -16,16 -16,14 +16,17 @@@
*/
package org.apache.accumulo.test;
+ import static com.google.common.base.Charsets.UTF_8;
+
+import java.io.IOException;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index 7a848a3,125ef5f..16f0b3f
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@@ -16,11 -16,11 +16,12 @@@
*/
package org.apache.accumulo.test;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.util.ArrayList;
+import java.util.List;
import java.util.Map.Entry;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.AccumuloException;
@@@ -101,8 -97,8 +102,8 @@@ public class TestMultiTableIngest
// populate
for (int i = 0; i < opts.count; i++) {
- Mutation m = new Mutation(new Text(String.format("%05d", i)));
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
+ m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(UTF_8)));
b.getBatchWriter(tableNames.get(i % tableNames.size())).addMutation(m);
}
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousMoru.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
index 6082ebe,5962220..4f88c1b
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousWalk.java
@@@ -26,7 -28,7 +28,6 @@@ import java.util.Map.Entry
import java.util.Random;
import java.util.zip.CRC32;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
index 4bd07ca,6124a3d..11d19c5
--- a/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/UndefinedAnalyzer.java
@@@ -82,9 -82,9 +83,9 @@@ public class UndefinedAnalyzer
parseLog(log);
}
}
-
+
private void parseLog(File log) throws Exception {
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), Constants.UTF8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(log), UTF_8));
String line;
TreeMap<Long,Long> tm = null;
try {
@@@ -173,10 -172,10 +174,10 @@@
SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM");
String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + "";
String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + "";
-
+
for (File masterLog : masterLogs) {
-
+
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), Constants.UTF8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(masterLog), UTF_8));
String line;
try {
while ((line = reader.readLine()) != null) {
@@@ -256,11 -252,11 +257,11 @@@
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
BatchScannerOpts bsOpts = new BatchScannerOpts();
- opts.parseArgs(UndefinedAnalyzer.class.getName(), args, opts);
-
+ opts.parseArgs(UndefinedAnalyzer.class.getName(), args, bsOpts);
+
List<UndefinedNode> undefs = new ArrayList<UndefinedNode>();
-
+
- BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, Constants.UTF8));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, UTF_8));
String line;
while ((line = reader.readLine()) != null) {
String[] tokens = line.split("\\s");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index f26c8d7,166e2ad..949fc52
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -16,6 -16,10 +16,8 @@@
*/
package org.apache.accumulo.test.functional;
+ import static com.google.common.base.Charsets.UTF_8;
+
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index cb17340,33ef0b5..72efee3
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@@ -24,7 -26,7 +26,6 @@@ import java.util.Random
import java.util.TreeSet;
import java.util.UUID;
--import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
@@@ -104,10 -100,10 +105,10 @@@ public class MetadataBatchScanTest
String dir = "/t-" + UUID.randomUUID();
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
- Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
for (int i = 0; i < 5; i++) {
- mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
- mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
++ mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8)));
}
bw.addMutation(mut);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
index 5756934,0191b50..e4ce3f3
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Module.java
@@@ -41,9 -37,7 +43,8 @@@ import javax.xml.parsers.DocumentBuilde
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
- import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.log4j.Level;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@@ -201,163 -193,109 +202,163 @@@ public class Module extends Node
if (fixture != null) {
fixture.setUp(state);
}
-
- Node initNode = getNode(initNodeId);
-
- boolean test = false;
- if (initNode instanceof Test) {
- startTimer(initNode);
- test = true;
- }
- initNode.visit(state, getProps(initNodeId));
- if (test)
- stopTimer(initNode);
-
- state.visitedNode();
- // update aliases
- Set<String> aliases;
- if ((aliases = aliasMap.get(initNodeId)) != null)
- for (String alias : aliases) {
- ((Alias) nodes.get(alias)).update(initNodeId);
- }
-
- String curNodeId = initNodeId;
- int numHops = 0;
- long startTime = System.currentTimeMillis() / 1000;
- while (true) {
- // check if END state was reached
- if (curNodeId.equalsIgnoreCase("END")) {
- log.debug("reached END state");
- break;
- }
- // check if maxSec was reached
- long curTime = System.currentTimeMillis() / 1000;
- if ((curTime - startTime) > maxSec) {
- log.debug("reached maxSec(" + maxSec + ")");
- break;
- }
- // check if maxHops was reached
- if (numHops > maxHops) {
- log.debug("reached maxHops(" + maxHops + ")");
- break;
- }
- numHops++;
-
- if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
- throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
- }
- AdjList adj = adjMap.get(curNodeId);
- String nextNodeId = adj.randomNeighbor();
- Node nextNode = getNode(nextNodeId);
- if (nextNode instanceof Alias) {
- nextNodeId = ((Alias) nextNode).getTargetId();
- nextNode = ((Alias) nextNode).get();
+
+ ExecutorService service = new SimpleThreadPool(1, "RandomWalk Runner");
+
+ try {
+ Node initNode = getNode(initNodeId);
+
+ boolean test = false;
+ if (initNode instanceof Test) {
+ startTimer(initNode);
+ test = true;
}
- Properties nodeProps = getProps(nextNodeId);
- try {
- test = false;
- if (nextNode instanceof Test) {
- startTimer(nextNode);
- test = true;
+ initNode.visit(state, getProps(initNodeId));
+ if (test)
+ stopTimer(initNode);
+
+ state.visitedNode();
+ // update aliases
+ Set<String> aliases;
+ if ((aliases = aliasMap.get(initNodeId)) != null)
+ for (String alias : aliases) {
+ ((Alias) nodes.get(alias)).update(initNodeId);
}
- nextNode.visit(state, nodeProps);
- if (test)
- stopTimer(nextNode);
- } catch (Exception e) {
- log.debug("Connector belongs to user: " + state.getConnector().whoami());
- log.debug("Exception occured at: " + System.currentTimeMillis());
- log.debug("Properties for node: " + nextNodeId);
- for (Entry<Object,Object> entry : nodeProps.entrySet()) {
- log.debug(" " + entry.getKey() + ": " + entry.getValue());
+
+ String curNodeId = initNodeId;
+ int numHops = 0;
+ long startTime = System.currentTimeMillis() / 1000;
+ while (true) {
+ // check if END state was reached
+ if (curNodeId.equalsIgnoreCase("END")) {
+ log.debug("reached END state");
+ break;
}
- log.debug("Overall Properties");
- for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
- log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ // check if maxSec was reached
+ long curTime = System.currentTimeMillis() / 1000;
+ if ((curTime - startTime) > maxSec) {
+ log.debug("reached maxSec(" + maxSec + ")");
+ break;
}
- log.debug("State information");
- for (String key : new TreeSet<String>(state.getMap().keySet())) {
- Object value = state.getMap().get(key);
- String logMsg = " " + key + ": ";
- if (value == null)
- logMsg += "null";
- else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
- logMsg += value;
- else if (value instanceof byte[])
- logMsg += new String((byte[])value, UTF_8);
- else if (value instanceof PasswordToken)
- logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
- else
- logMsg += value.getClass()+ " - " + value;
-
- log.debug(logMsg);
+
+ // The number of seconds before the test should exit
+ long secondsRemaining = maxSec - (curTime - startTime);
+
+ // check if maxHops was reached
+ if (numHops > maxHops) {
+ log.debug("reached maxHops(" + maxHops + ")");
+ break;
}
- throw new Exception("Error running node " + nextNodeId, e);
- }
- state.visitedNode();
-
- // update aliases
- if ((aliases = aliasMap.get(curNodeId)) != null)
- for (String alias : aliases) {
- ((Alias) nodes.get(alias)).update(curNodeId);
+ numHops++;
+
+ if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
+ throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
+ }
+ AdjList adj = adjMap.get(curNodeId);
+ String nextNodeId = adj.randomNeighbor();
+ final Node nextNode;
+ Node nextNodeOrAlias = getNode(nextNodeId);
+ if (nextNodeOrAlias instanceof Alias) {
+ nextNodeId = ((Alias) nextNodeOrAlias).getTargetId();
+ nextNode = ((Alias) nextNodeOrAlias).get();
+ } else {
+ nextNode = nextNodeOrAlias;
+ }
+ final Properties nodeProps = getProps(nextNodeId);
+ try {
+ test = false;
+ if (nextNode instanceof Test) {
+ startTimer(nextNode);
+ test = true;
+ }
+
+ // Wrap the visit of the next node in the module in a callable that returns a thrown exception
+ FutureTask<Exception> task = new FutureTask<Exception>(new Callable<Exception>() {
+
+ @Override
+ public Exception call() throws Exception {
+ try {
+ nextNode.visit(state, nodeProps);
+ return null;
+ } catch (Exception e) {
+ return e;
+ }
+ }
+
+ });
+
+ // Run the task (should execute immediately)
+ service.submit(task);
+
+ Exception nodeException;
+ try {
+ // Bound the time we'll wait for the node to complete
+ nodeException = task.get(secondsRemaining, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ log.warn("Interrupted waiting for " + nextNode.getClass().getSimpleName() + " to complete. Exiting.", e);
+ break;
+ } catch (ExecutionException e) {
+ log.error("Caught error executing " + nextNode.getClass().getSimpleName(), e);
+ throw e;
+ } catch (TimeoutException e) {
+ log.info("Timed out waiting for " + nextNode.getClass().getSimpleName() + " to complete (waited " + secondsRemaining + " seconds). Exiting.", e);
+ break;
+ }
+
+ // The RandomWalk node throw an Exception that that Callable handed back
+ // Throw it and let the Module perform cleanup
+ if (null != nodeException) {
+ throw nodeException;
+ }
+
+ if (test)
+ stopTimer(nextNode);
+ } catch (Exception e) {
+ log.debug("Connector belongs to user: " + state.getConnector().whoami());
+ log.debug("Exception occured at: " + System.currentTimeMillis());
+ log.debug("Properties for node: " + nextNodeId);
+ for (Entry<Object,Object> entry : nodeProps.entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
+ log.debug("Overall Properties");
+ for (Entry<Object,Object> entry : state.getProperties().entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
+ log.debug("State information");
+ for (String key : new TreeSet<String>(state.getMap().keySet())) {
+ Object value = state.getMap().get(key);
+ String logMsg = " " + key + ": ";
+ if (value == null)
+ logMsg += "null";
+ else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
+ logMsg += value;
+ else if (value instanceof byte[])
- logMsg += new String((byte[]) value, Constants.UTF8);
++ logMsg += new String((byte[]) value, UTF_8);
+ else if (value instanceof PasswordToken)
- logMsg += new String(((PasswordToken) value).getPassword(), Constants.UTF8);
++ logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
+ else
+ logMsg += value.getClass() + " - " + value;
+
+ log.debug(logMsg);
+ }
+ throw new Exception("Error running node " + nextNodeId, e);
}
-
- curNodeId = nextNodeId;
+ state.visitedNode();
+
+ // update aliases
+ if ((aliases = aliasMap.get(curNodeId)) != null)
+ for (String alias : aliases) {
+ ((Alias) nodes.get(alias)).update(curNodeId);
+ }
+
+ curNodeId = nextNodeId;
+ }
+ } finally {
+ if (null != service) {
+ service.shutdownNow();
+ }
}
-
+
if (teardown && (fixture != null)) {
log.debug("tearing down module");
fixture.tearDown(state);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
index 1704e49,05cc4f0..67438d9
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/bulk/BulkMinusOne.java
@@@ -20,10 -21,10 +21,10 @@@ import static com.google.common.base.Ch
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.test.randomwalk.State;
-public class BulkMinusOne extends BulkTest {
-
+public class BulkMinusOne extends BulkImportTest {
+
- private static final Value negOne = new Value("-1".getBytes(Constants.UTF8));
+ private static final Value negOne = new Value("-1".getBytes(UTF_8));
-
+
@Override
protected void runLater(State state) throws Exception {
log.info("Decrementing");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
index f002274,7f9a218..3dcff6e
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BatchWrite.java
@@@ -51,10 -52,10 +52,10 @@@ public class BatchWrite extends Test
try {
int numRows = rand.nextInt(100000);
for (int i = 0; i < numRows; i++) {
- Mutation m = new Mutation(String.format("%016x", (rand.nextLong() & 0x7fffffffffffffffl)));
- long val = (rand.nextLong() & 0x7fffffffffffffffl);
+ Mutation m = new Mutation(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl));
+ long val = rand.nextLong() & 0x7fffffffffffffffl;
for (int j = 0; j < 10; j++) {
- m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(Constants.UTF8)));
+ m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(UTF_8)));
}
bw.addMutation(m);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
index c8da3d8,c227a11..b3519b6
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/StopTabletServer.java
@@@ -47,8 -50,8 +49,8 @@@ public class StopTabletServer extends T
Collections.sort(children);
Stat stat = new Stat();
byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
- if (!"master".equals(new String(data, Constants.UTF8))) {
+ if (!"master".equals(new String(data, UTF_8))) {
- result.add(new TServerInstance(AddressUtil.parseAddress(child, Property.TSERV_CLIENTPORT), stat.getEphemeralOwner()));
+ result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
}
}
} catch (KeeperException.NoNodeException ex) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/image/Write.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/security/TableOp.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/sequential/Write.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
[41/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
index 103cfe8,0000000..547ccc7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java
@@@ -1,1601 -1,0 +1,1603 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.NamespaceExistsException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.admin.DiskUsage;
+import org.apache.accumulo.core.client.admin.FindMax;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.thrift.ClientService;
+import org.apache.accumulo.core.client.impl.thrift.ClientService.Client;
+import org.apache.accumulo.core.client.impl.thrift.TDiskUsage;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.FateOperation;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.metadata.MetadataServicer;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.util.ArgumentChecker;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+
+public class TableOperationsImpl extends TableOperationsHelper {
+ private Instance instance;
+ private Credentials credentials;
+
+ public static final String CLONE_EXCLUDE_PREFIX = "!";
+
+ private static final Logger log = Logger.getLogger(TableOperations.class);
+
+ /**
+ * @param instance
+ * the connection information for this instance
+ * @param credentials
+ * the username/password for this connection
+ */
+ public TableOperationsImpl(Instance instance, Credentials credentials) {
+ ArgumentChecker.notNull(instance, credentials);
+ this.instance = instance;
+ this.credentials = credentials;
+ }
+
+ /**
+ * Retrieve a list of tables in Accumulo.
+ *
+ * @return List of tables in accumulo
+ */
+ @Override
+ public SortedSet<String> list() {
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Fetching list of tables...");
+ TreeSet<String> tableNames = new TreeSet<String>(Tables.getNameToIdMap(instance).keySet());
+ opTimer.stop("Fetched " + tableNames.size() + " table names in %DURATION%");
+ return tableNames;
+ }
+
+ /**
+ * A method to check if a table exists in Accumulo.
+ *
+ * @param tableName
+ * the name of the table
+ * @return true if the table exists
+ */
+ @Override
+ public boolean exists(String tableName) {
+ ArgumentChecker.notNull(tableName);
+ if (tableName.equals(MetadataTable.NAME) || tableName.equals(RootTable.NAME))
+ return true;
+
+ OpTimer opTimer = new OpTimer(log, Level.TRACE).start("Checking if table " + tableName + " exists...");
+ boolean exists = Tables.getNameToIdMap(instance).containsKey(tableName);
+ opTimer.stop("Checked existance of " + exists + " in %DURATION%");
+ return exists;
+ }
+
+ /**
+ * Create a table with no special configuration
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableExistsException
+ * if the table already exists
+ */
+ @Override
+ public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, true, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ create(tableName, limitVersion, TimeType.MILLIS);
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param timeType
+ * specifies logical or real-time based time recording for entries in the table
+ * @param limitVersion
+ * Enables/disables the versioning iterator, which will limit the number of Key versions kept.
+ */
+ @Override
+ public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
+ ArgumentChecker.notNull(tableName, timeType);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(timeType.name().getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(timeType.name().getBytes(UTF_8)));
+
+ Map<String,String> opts;
+ if (limitVersion)
+ opts = IteratorUtil.generateInitialTableProperties(limitVersion);
+ else
+ opts = Collections.emptyMap();
+
+ try {
+ doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_CREATE, args, opts);
+ } catch (TableNotFoundException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ private long beginFateOperation() throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.beginFateOperation(Tracer.traceInfo(), credentials.toThrift(instance));
+ } catch (TTransportException tte) {
+ log.debug("Failed to call beginFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ // This method is for retrying in the case of network failures; anything else it passes to the caller to deal with
+ private void executeFateOperation(long opid, FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean autoCleanUp)
+ throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.executeFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid, op, args, opts, autoCleanUp);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call executeFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private String waitForFateOperation(long opid) throws ThriftSecurityException, TException, ThriftTableOperationException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ return client.waitForFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
+ } catch (TTransportException tte) {
+ log.debug("Failed to call waitForFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ private void finishFateOperation(long opid) throws ThriftSecurityException, TException {
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.finishFateOperation(Tracer.traceInfo(), credentials.toThrift(instance), opid);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call finishFateOperation(), retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+ String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts) throws AccumuloSecurityException, TableExistsException,
+ TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
+ return doFateOperation(op, args, opts, true);
+ }
+
+ String doFateOperation(FateOperation op, List<ByteBuffer> args, Map<String,String> opts, boolean wait) throws AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, AccumuloException, NamespaceExistsException, NamespaceNotFoundException {
+ Long opid = null;
+
+ try {
+ opid = beginFateOperation();
+ executeFateOperation(opid, op, args, opts, !wait);
+ if (!wait) {
+ opid = null;
+ return null;
+ }
+ String ret = waitForFateOperation(opid);
+ return ret;
+ } catch (ThriftSecurityException e) {
+ String tableName = ByteBufferUtil.toString(args.get(0));
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(null, tableName, "Target table does not exist");
+ case NAMESPACE_DOESNT_EXIST:
+ throw new NamespaceNotFoundException(null, tableName, "Target namespace does not exist");
+ default:
+ String tableInfo = Tables.getPrintableTableInfoFromName(instance, tableName);
+ throw new AccumuloSecurityException(e.user, e.code, tableInfo, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case EXISTS:
+ throw new TableExistsException(e);
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_EXISTS:
+ throw new NamespaceExistsException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new NamespaceNotFoundException(e);
+ case OFFLINE:
+ throw new TableOfflineException(instance, null);
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e.getMessage(), e);
+ } finally {
+ Tables.clearCache(instance);
+ // always finish table op, even when exception
+ if (opid != null)
+ try {
+ finishFateOperation(opid);
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class SplitEnv {
+ private String tableName;
+ private String tableId;
+ private ExecutorService executor;
+ private CountDownLatch latch;
+ private AtomicReference<Exception> exception;
+
+ SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
+ this.tableName = tableName;
+ this.tableId = tableId;
+ this.executor = executor;
+ this.latch = latch;
+ this.exception = exception;
+ }
+ }
+
+ private class SplitTask implements Runnable {
+
+ private List<Text> splits;
+ private SplitEnv env;
+
+ SplitTask(SplitEnv env, List<Text> splits) {
+ this.env = env;
+ this.splits = splits;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (env.exception.get() != null)
+ return;
+
+ if (splits.size() <= 2) {
+ addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
+ for (int i = 0; i < splits.size(); i++)
+ env.latch.countDown();
+ return;
+ }
+
+ int mid = splits.size() / 2;
+
+ // split the middle split point to ensure that child task split different tablets and can therefore
+ // run in parallel
+ addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
+ env.latch.countDown();
+
+ env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+ env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
+
+ } catch (Exception e) {
+ env.exception.compareAndSet(null, e);
+ }
+ }
+
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param partitionKeys
+ * a sorted set of row key values to pre-split the table on
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ String tableId = Tables.getTableId(instance, tableName);
+
+ List<Text> splits = new ArrayList<Text>(partitionKeys);
+ // should be sorted because we copied from a sorted set, but that makes assumptions about
+ // how the copy was done so resort to be sure.
+ Collections.sort(splits);
+
+ CountDownLatch latch = new CountDownLatch(splits.size());
+ AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
+
+ ExecutorService executor = Executors.newFixedThreadPool(16, new NamingThreadFactory("addSplits"));
+ try {
+ executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
+
+ while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+ if (exception.get() != null) {
+ executor.shutdownNow();
+ Exception excep = exception.get();
+ if (excep instanceof TableNotFoundException)
+ throw (TableNotFoundException) excep;
+ else if (excep instanceof AccumuloException)
+ throw (AccumuloException) excep;
+ else if (excep instanceof AccumuloSecurityException)
+ throw (AccumuloSecurityException) excep;
+ else if (excep instanceof RuntimeException)
+ throw (RuntimeException) excep;
+ else
+ throw new RuntimeException(excep);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloServerException {
+ TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(tableId));
+
+ for (Text split : partitionKeys) {
+ boolean successful = false;
+ int attempt = 0;
+ long locationFailures = 0;
+
+ while (!successful) {
+
+ if (attempt > 0)
+ UtilWaitThread.sleep(100);
+
+ attempt++;
+
+ TabletLocation tl = tabLocator.locateTablet(credentials, split, false, false);
+
+ if (tl == null) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+ continue;
+ }
+
+ try {
+ TabletClientService.Client client = ThriftUtil.getTServerClient(tl.tablet_location, ServerConfigurationUtil.getConfiguration(instance));
+ try {
+ OpTimer opTimer = null;
+ if (log.isTraceEnabled())
+ opTimer = new OpTimer(log, Level.TRACE).start("Splitting tablet " + tl.tablet_extent + " on " + tl.tablet_location + " at " + split);
+
+ client.splitTablet(Tracer.traceInfo(), credentials.toThrift(instance), tl.tablet_extent.toThrift(), TextUtil.getByteBuffer(split));
+
+ // just split it, might as well invalidate it in the cache
+ tabLocator.invalidateCache(tl.tablet_extent);
+
+ if (opTimer != null)
+ opTimer.stop("Split tablet in %DURATION%");
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+
+ } catch (TApplicationException tae) {
+ throw new AccumuloServerException(tl.tablet_location, tae);
+ } catch (TTransportException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ } catch (ThriftSecurityException e) {
+ Tables.clearCache(instance);
+ if (!Tables.exists(instance, tableId))
+ throw new TableNotFoundException(tableId, tableName, null);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (NotServingTabletException e) {
+ // Do not silently spin when we repeatedly fail to get the location for a tablet
+ locationFailures++;
+ if (5 == locationFailures || 0 == locationFailures % 50) {
+ log.warn("Having difficulty locating hosting tabletserver for split " + split + " on table " + tableName + ". Seen " + locationFailures
+ + " failures.");
+ }
+
+ tabLocator.invalidateCache(tl.tablet_extent);
+ continue;
+ } catch (TException e) {
+ tabLocator.invalidateCache(tl.tablet_location);
+ continue;
+ }
+
+ successful = true;
+ }
+ }
+ }
+
+ @Override
+ public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ end == null ? EMPTY : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_MERGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ end == null ? EMPTY : TextUtil.getByteBuffer(end));
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE_RANGE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @return the split points (end-row names) for the table's current split profile
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException {
+
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ TreeMap<KeyExtent,String> tabletLocations = new TreeMap<KeyExtent,String>();
+
+ while (true) {
+ try {
+ tabletLocations.clear();
+ // the following method throws AccumuloException for some conditions that should be retried
+ MetadataServicer.forTableId(instance, credentials, tableId).getTabletLocations(tabletLocations);
+ break;
+ } catch (AccumuloSecurityException ase) {
+ throw ase;
+ } catch (Exception e) {
+ if (!Tables.exists(instance, tableId)) {
+ throw new TableNotFoundException(tableId, tableName, null);
+ }
+
+ if (e instanceof RuntimeException && e.getCause() instanceof AccumuloSecurityException) {
+ throw (AccumuloSecurityException) e.getCause();
+ }
+
+ log.info(e.getMessage() + " ... retrying ...");
+ UtilWaitThread.sleep(3000);
+ }
+ }
+
+ ArrayList<Text> endRows = new ArrayList<Text>(tabletLocations.size());
+
+ for (KeyExtent ke : tabletLocations.keySet())
+ if (ke.getEndRow() != null)
+ endRows.add(ke.getEndRow());
+
+ return endRows;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName) throws TableNotFoundException {
+ try {
+ return listSplits(tableName);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return the split points (end-row names) for the table's current split profile, grouped into fewer splits so as not to exceed maxSplits
+ */
+ @Override
+ public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException {
+ Collection<Text> endRows = listSplits(tableName);
+
+ if (endRows.size() <= maxSplits)
+ return endRows;
+
+ double r = (maxSplits + 1) / (double) (endRows.size());
+ double pos = 0;
+
+ ArrayList<Text> subset = new ArrayList<Text>(maxSplits);
+
+ int j = 0;
+ for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
+ pos += r;
+ while (pos > 1) {
+ subset.add(((ArrayList<Text>) endRows).get(i));
+ j++;
+ pos -= 1;
+ }
+ }
+
+ return subset;
+ }
+
+ @Deprecated
+ @Override
+ public Collection<Text> getSplits(String tableName, int maxSplits) throws TableNotFoundException {
+ try {
+ return listSplits(tableName, maxSplits);
+ } catch (AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Delete a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ }
+
+ @Override
+ public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
+ throws AccumuloSecurityException, TableNotFoundException, AccumuloException, TableExistsException {
+
+ ArgumentChecker.notNull(srcTableName, newTableName);
+
+ String srcTableId = Tables.getTableId(instance, srcTableName);
+
+ if (flush)
+ _flush(srcTableId, null, null, true);
+
+ if (propertiesToExclude == null)
+ propertiesToExclude = Collections.emptySet();
+
+ if (propertiesToSet == null)
+ propertiesToSet = Collections.emptyMap();
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(srcTableId.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ for (Entry<String,String> entry : propertiesToSet.entrySet()) {
+ if (entry.getKey().startsWith(CLONE_EXCLUDE_PREFIX))
+ throw new IllegalArgumentException("Property can not start with " + CLONE_EXCLUDE_PREFIX);
+ opts.put(entry.getKey(), entry.getValue());
+ }
+
+ for (String prop : propertiesToExclude) {
+ opts.put(CLONE_EXCLUDE_PREFIX + prop, "");
+ }
+
+ doTableFateOperation(newTableName, AccumuloException.class, FateOperation.TABLE_CLONE, args, opts);
+ }
+
+ /**
+ * Rename a table
+ *
+ * @param oldTableName
+ * the old table name
+ * @param newTableName
+ * the new table name
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the old table name does not exist
+ * @throws TableExistsException
+ * if the new table name already exists
+ */
+ @Override
+ public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException,
+ TableExistsException {
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(newTableName.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(oldTableName.getBytes(UTF_8)), ByteBuffer.wrap(newTableName.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+ doTableFateOperation(oldTableName, TableNotFoundException.class, FateOperation.TABLE_RENAME, args, opts);
+ }
+
+ /**
+ * @deprecated since 1.4 {@link #flush(String, Text, Text, boolean)}
+ */
+ @Override
+ @Deprecated
+ public void flush(String tableName) throws AccumuloException, AccumuloSecurityException {
+ try {
+ flush(tableName, null, null, false);
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Flush a table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+
+ String tableId = Tables.getTableId(instance, tableName);
+ _flush(tableId, start, end, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
+ AccumuloException {
+ compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
+ }
+
+ @Override
+ public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName);
+ ByteBuffer EMPTY = ByteBuffer.allocate(0);
+
+ String tableId = Tables.getTableId(instance, tableName);
+
+ if (flush)
+ _flush(tableId, start, end, true);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)), start == null ? EMPTY : TextUtil.getByteBuffer(start),
+ end == null ? EMPTY : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doFateOperation(FateOperation.TABLE_COMPACT, args, opts, wait);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ } catch (NamespaceExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ } catch (NamespaceNotFoundException e) {
+ throw new TableNotFoundException(null, tableName, "Namespace not found", e);
+ }
+ }
+
+ @Override
+ public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException {
+ String tableId = Tables.getTableId(instance, tableName);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
+
+ Map<String,String> opts = new HashMap<String,String>();
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_CANCEL_COMPACT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ }
+
+ private void _flush(String tableId, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ try {
+ long flushID;
+
+ // used to pass the table name. but the tableid associated with a table name could change between calls.
+ // so pass the tableid to both calls
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ flushID = client.initiateFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+
+ while (true) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(instance);
+ client.waitForFlush(Tracer.traceInfo(), credentials.toThrift(instance), tableId, TextUtil.getByteBuffer(start), TextUtil.getByteBuffer(end), flushID,
+ wait ? Long.MAX_VALUE : 1);
+ break;
+ } catch (TTransportException tte) {
+ log.debug("Failed to call initiateFlush, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } finally {
+ MasterClient.close(client);
+ }
+ }
+ } catch (ThriftSecurityException e) {
+ switch (e.getCode()) {
+ case TABLE_DOESNT_EXIST:
+ throw new TableNotFoundException(tableId, null, e.getMessage(), e);
+ default:
+ log.debug("flush security exception on table id " + tableId);
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ }
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Sets a property on a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @param value
+ * the value to set a per-table property to
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void setProperty(final String tableName, final String property, final String value) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property, value);
+ try {
+ MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.setTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property, value);
+ }
+ });
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Removes a property from a table
+ *
+ * @param tableName
+ * the name of the table
+ * @param property
+ * the name of a per-table property
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ */
+ @Override
+ public void removeProperty(final String tableName, final String property) throws AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, property);
+ try {
+ MasterClient.executeTable(instance, new ClientExec<MasterClientService.Client>() {
+ @Override
+ public void execute(MasterClientService.Client client) throws Exception {
+ client.removeTableProperty(Tracer.traceInfo(), credentials.toThrift(instance), tableName, property);
+ }
+ });
+ } catch (TableNotFoundException e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ /**
+ * Gets properties of a table
+ *
+ * @param tableName
+ * the name of the table
+ * @return all properties visible by this table (system and per-table properties)
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Iterable<Entry<String,String>> getProperties(final String tableName) throws AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() {
+ @Override
+ public Map<String,String> execute(ClientService.Client client) throws Exception {
+ return client.getTableConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), tableName);
+ }
+ }).entrySet();
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+
+ }
+
+ /**
+ * Sets a tables locality groups. A tables locality groups can be changed at any time.
+ *
+ * @param tableName
+ * the name of the table
+ * @param groups
+ * mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public void setLocalityGroups(String tableName, Map<String,Set<Text>> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ // ensure locality groups do not overlap
+ HashSet<Text> all = new HashSet<Text>();
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+
+ if (!Collections.disjoint(all, entry.getValue())) {
+ throw new IllegalArgumentException("Group " + entry.getKey() + " overlaps with another group");
+ }
+
+ all.addAll(entry.getValue());
+ }
+
+ for (Entry<String,Set<Text>> entry : groups.entrySet()) {
+ Set<Text> colFams = entry.getValue();
+ String value = LocalityGroupUtil.encodeColumnFamilies(colFams);
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUP_PREFIX + entry.getKey(), value);
+ }
+
+ try {
+ setProperty(tableName, Property.TABLE_LOCALITY_GROUPS.getKey(), StringUtil.join(groups.keySet(), ","));
+ } catch (AccumuloException e) {
+ if (e.getCause() instanceof TableNotFoundException)
+ throw (TableNotFoundException) e.getCause();
+ throw e;
+ }
+
+ // remove anything extraneous
+ String prefix = Property.TABLE_LOCALITY_GROUP_PREFIX.getKey();
+ for (Entry<String,String> entry : getProperties(tableName)) {
+ String property = entry.getKey();
+ if (property.startsWith(prefix)) {
+ // this property configures a locality group, find out which
+ // one:
+ String[] parts = property.split("\\.");
+ String group = parts[parts.length - 1];
+
+ if (!groups.containsKey(group)) {
+ removeProperty(tableName, property);
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ * Gets the locality groups currently set for a table.
+ *
+ * @param tableName
+ * the name of the table
+ * @return mapping of locality group names to column families in the locality group
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Map<String,Set<Text>> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException {
+ AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
+ Map<String,Set<ByteSequence>> groups = LocalityGroupUtil.getLocalityGroups(conf);
+
+ Map<String,Set<Text>> groups2 = new HashMap<String,Set<Text>>();
+ for (Entry<String,Set<ByteSequence>> entry : groups.entrySet()) {
+
+ HashSet<Text> colFams = new HashSet<Text>();
+
+ for (ByteSequence bs : entry.getValue()) {
+ colFams.add(new Text(bs.toArray()));
+ }
+
+ groups2.put(entry.getKey(), colFams);
+ }
+
+ return groups2;
+ }
+
+ /**
+ * @param tableName
+ * the name of the table
+ * @param range
+ * a range to split
+ * @param maxSplits
+ * the maximum number of splits
+ * @return the range, split into smaller ranges that fall on boundaries of the table's split points as evenly as possible
+ * @throws AccumuloException
+ * if a general error occurs
+ * @throws AccumuloSecurityException
+ * if the user does not have permission
+ * @throws TableNotFoundException
+ * if the table does not exist
+ */
+ @Override
+ public Set<Range> splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException,
+ TableNotFoundException {
+ ArgumentChecker.notNull(tableName, range);
+ if (maxSplits < 1)
+ throw new IllegalArgumentException("maximum splits must be >= 1");
+ if (maxSplits == 1)
+ return Collections.singleton(range);
+
+ Random random = new Random();
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+ String tableId = Tables.getTableId(instance, tableName);
+ TabletLocator tl = TabletLocator.getLocator(instance, new Text(tableId));
+ // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
+ tl.invalidateCache();
+ while (!tl.binRanges(credentials, Collections.singletonList(range), binnedRanges).isEmpty()) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(instance, tableId);
+
+ log.warn("Unable to locate bins for specified range. Retrying.");
+ // sleep randomly between 100 and 200ms
+ UtilWaitThread.sleep(100 + random.nextInt(100));
+ binnedRanges.clear();
+ tl.invalidateCache();
+ }
+
+ // group key extents to get <= maxSplits
+ LinkedList<KeyExtent> unmergedExtents = new LinkedList<KeyExtent>();
+ List<KeyExtent> mergedExtents = new ArrayList<KeyExtent>();
+
+ for (Map<KeyExtent,List<Range>> map : binnedRanges.values())
+ unmergedExtents.addAll(map.keySet());
+
+ // the sort method is efficient for linked list
+ Collections.sort(unmergedExtents);
+
+ while (unmergedExtents.size() + mergedExtents.size() > maxSplits) {
+ if (unmergedExtents.size() >= 2) {
+ KeyExtent first = unmergedExtents.removeFirst();
+ KeyExtent second = unmergedExtents.removeFirst();
+ first.setEndRow(second.getEndRow());
+ mergedExtents.add(first);
+ } else {
+ mergedExtents.addAll(unmergedExtents);
+ unmergedExtents.clear();
+ unmergedExtents.addAll(mergedExtents);
+ mergedExtents.clear();
+ }
+
+ }
+
+ mergedExtents.addAll(unmergedExtents);
+
+ Set<Range> ranges = new HashSet<Range>();
+ for (KeyExtent k : mergedExtents)
+ ranges.add(k.toDataRange().clip(range));
+
+ return ranges;
+ }
+
+ // TODO Remove deprecation warning surppression when Hadoop1 support is dropped
+ @SuppressWarnings("deprecation")
+ private Path checkPath(String dir, String kind, String type) throws IOException, AccumuloException {
+ Path ret;
+ FileSystem fs = VolumeConfiguration.getVolume(dir, CachedConfiguration.getInstance(), ServerConfigurationUtil.getConfiguration(instance)).getFileSystem();
+
+ if (dir.contains(":")) {
+ ret = new Path(dir);
+ } else {
+ ret = fs.makeQualified(new Path(dir));
+ }
+
+ if (!fs.exists(ret))
+ throw new AccumuloException(kind + " import " + type + " directory " + dir + " does not exist!");
+
+ if (!fs.getFileStatus(ret).isDir()) {
+ throw new AccumuloException(kind + " import " + type + " directory " + dir + " is not a directory!");
+ }
+
+ if (type.equals("failure")) {
+ FileStatus[] listStatus = fs.listStatus(ret);
+ if (listStatus != null && listStatus.length != 0) {
+ throw new AccumuloException("Bulk import failure directory " + ret + " is not empty");
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloSecurityException,
+ TableNotFoundException, AccumuloException {
+ ArgumentChecker.notNull(tableName, dir, failureDir);
+ // check for table existance
+ Tables.getTableId(instance, tableName);
+
+ Path dirPath = checkPath(dir, "Bulk", "");
+ Path failPath = checkPath(failureDir, "Bulk", "failure");
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(dirPath.toString().getBytes(Constants.UTF8)),
- ByteBuffer.wrap(failPath.toString().getBytes(Constants.UTF8)), ByteBuffer.wrap((setTime + "").getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(dirPath.toString().getBytes(UTF_8)),
++ ByteBuffer.wrap(failPath.toString().getBytes(UTF_8)), ByteBuffer.wrap((setTime + "").getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_BULK_IMPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ private void waitForTableStateTransition(String tableId, TableState expectedState) throws AccumuloException, TableNotFoundException,
+ AccumuloSecurityException {
+
+ Text startRow = null;
+ Text lastRow = null;
+
+ while (true) {
+
+ if (Tables.getTableState(instance, tableId) != expectedState) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != expectedState) {
+ if (!Tables.exists(instance, tableId))
+ throw new TableDeletedException(tableId);
+ throw new AccumuloException("Unexpected table state " + tableId + " " + Tables.getTableState(instance, tableId) + " != " + expectedState);
+ }
+ }
+
+ Range range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ if (startRow == null || lastRow == null)
+ range = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
+ else
+ range = new Range(startRow, lastRow);
+
+ String metaTable = MetadataTable.NAME;
+ if (tableId.equals(MetadataTable.ID))
+ metaTable = RootTable.NAME;
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(metaTable, Authorizations.EMPTY);
+ scanner = new IsolatedScanner(scanner);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.setRange(range);
+
+ RowIterator rowIter = new RowIterator(scanner);
+
+ KeyExtent lastExtent = null;
+
+ int total = 0;
+ int waitFor = 0;
+ int holes = 0;
+ Text continueRow = null;
+ MapCounter<String> serverCounts = new MapCounter<String>();
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+
+ total++;
+
+ KeyExtent extent = null;
+ String future = null;
+ String current = null;
+
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME))
+ future = entry.getValue().toString();
+
+ if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME))
+ current = entry.getValue().toString();
+
+ if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key))
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ if ((expectedState == TableState.ONLINE && current == null) || (expectedState == TableState.OFFLINE && (future != null || current != null))) {
+ if (continueRow == null)
+ continueRow = extent.getMetadataEntry();
+ waitFor++;
+ lastRow = extent.getMetadataEntry();
+
+ if (current != null)
+ serverCounts.increment(current, 1);
+ if (future != null)
+ serverCounts.increment(future, 1);
+ }
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ holes++;
+ }
+
+ lastExtent = extent;
+ }
+
+ if (continueRow != null) {
+ startRow = continueRow;
+ }
+
+ if (holes > 0 || total == 0) {
+ startRow = null;
+ lastRow = null;
+ }
+
+ if (waitFor > 0 || holes > 0 || total == 0) {
+ long waitTime;
+ long maxPerServer = 0;
+ if (serverCounts.size() > 0) {
+ maxPerServer = Collections.max(serverCounts.values());
+ waitTime = maxPerServer * 10;
+ } else
+ waitTime = waitFor * 10;
+ waitTime = Math.max(100, waitTime);
+ waitTime = Math.min(5000, waitTime);
+ log.trace("Waiting for " + waitFor + "(" + maxPerServer + ") tablets, startRow = " + startRow + " lastRow = " + lastRow + ", holes=" + holes
+ + " sleeping:" + waitTime + "ms");
+ UtilWaitThread.sleep(waitTime);
+ } else {
+ break;
+ }
+
+ }
+ }
+
+ @Override
+ public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ offline(tableName, false);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take offline
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ */
+ @Override
+ public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+
+ ArgumentChecker.notNull(tableName);
+ String tableId = Tables.getTableId(instance, tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_OFFLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ if (wait)
+ waitForTableStateTransition(tableId, TableState.OFFLINE);
+ }
+
+ @Override
+ public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ online(tableName, false);
+ }
+
+ /**
+ *
+ * @param tableName
+ * the table to take online
+ * @throws AccumuloException
+ * when there is a general accumulo error
+ * @throws AccumuloSecurityException
+ * when the user does not have the proper permissions
+ */
+ @Override
+ public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ String tableId = Tables.getTableId(instance, tableName);
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes(UTF_8)));
+ Map<String,String> opts = new HashMap<String,String>();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_ONLINE, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ if (wait)
+ waitForTableStateTransition(tableId, TableState.ONLINE);
+ }
+
+ /**
+ * Clears the tablet locator cache for a specified table
+ *
+ * @param tableName
+ * the name of the table
+ * @throws TableNotFoundException
+ * if table does not exist
+ */
+ @Override
+ public void clearLocatorCache(String tableName) throws TableNotFoundException {
+ ArgumentChecker.notNull(tableName);
+ TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(Tables.getTableId(instance, tableName)));
+ tabLocator.invalidateCache();
+ }
+
+ /**
+ * Get a mapping of table name to internal table id.
+ *
+ * @return the map from table name to internal table id
+ */
+ @Override
+ public Map<String,String> tableIdMap() {
+ return Tables.getNameToIdMap(instance);
+ }
+
+ @Override
+ public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, auths);
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, auths);
+ return FindMax.findMax(scanner, startRow, startInclusive, endRow, endInclusive);
+ }
+
+ @Override
+ public List<DiskUsage> getDiskUsage(Set<String> tableNames) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+
+ List<TDiskUsage> diskUsages = null;
+ while (diskUsages == null) {
+ Pair<String,Client> pair = null;
+ try {
+ // this operation may us a lot of memory... its likely that connections to tabletservers hosting metadata tablets will be cached, so do not use cached
+ // connections
+ pair = ServerClient.getConnection(instance, false);
+ diskUsages = pair.getSecond().getDiskUsage(tableNames, credentials.toThrift(instance));
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new TableNotFoundException(e.getTableName(), new NamespaceNotFoundException(e));
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.getUser(), e.getCode());
+ } catch (TTransportException e) {
+ // some sort of communication error occurred, retry
+ if (pair == null) {
+ log.debug("Disk usage request failed. Pair is null. Retrying request...", e);
+ } else {
+ log.debug("Disk usage request failed " + pair.getFirst() + ", retrying ... ", e);
+ }
+ UtilWaitThread.sleep(100);
+ } catch (TException e) {
+ // may be a TApplicationException which indicates error on the server side
+ throw new AccumuloException(e);
+ } finally {
+ // must always return thrift connection
+ if (pair != null)
+ ServerClient.close(pair.getSecond());
+ }
+ }
+
+ List<DiskUsage> finalUsages = new ArrayList<DiskUsage>();
+ for (TDiskUsage diskUsage : diskUsages) {
+ finalUsages.add(new DiskUsage(new TreeSet<String>(diskUsage.getTables()), diskUsage.getUsage()));
+ }
+
+ return finalUsages;
+ }
+
+ public static Map<String,String> getExportedProps(FileSystem fs, Path path) throws IOException {
+ HashMap<String,String> props = new HashMap<String,String>();
+
+ ZipInputStream zis = new ZipInputStream(fs.open(path));
+ try {
+ ZipEntry zipEntry;
+ while ((zipEntry = zis.getNextEntry()) != null) {
+ if (zipEntry.getName().equals(Constants.EXPORT_TABLE_CONFIG_FILE)) {
- BufferedReader in = new BufferedReader(new InputStreamReader(zis, Constants.UTF8));
++ BufferedReader in = new BufferedReader(new InputStreamReader(zis, UTF_8));
+ try {
+ String line;
+ while ((line = in.readLine()) != null) {
+ String sa[] = line.split("=", 2);
+ props.put(sa[0], sa[1]);
+ }
+ } finally {
+ in.close();
+ }
+
+ break;
+ }
+ }
+ } finally {
+ zis.close();
+ }
+ return props;
+ }
+
+ @Override
+ public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, importDir);
+
+ try {
+ importDir = checkPath(importDir, "Table", "").toString();
+ } catch (IOException e) {
+ throw new AccumuloException(e);
+ }
+
+ try {
+ FileSystem fs = new Path(importDir).getFileSystem(CachedConfiguration.getInstance());
+ Map<String,String> props = getExportedProps(fs, new Path(importDir, Constants.EXPORT_FILE));
+
+ for (Entry<String,String> entry : props.entrySet()) {
+ if (Property.isClassProperty(entry.getKey()) && !entry.getValue().contains(Constants.CORE_PACKAGE_NAME)) {
+ Logger.getLogger(this.getClass()).info(
+ "Imported table sets '" + entry.getKey() + "' to '" + entry.getValue() + "'. Ensure this class is on Accumulo classpath.");
+ }
+ }
+
+ } catch (IOException ioe) {
+ Logger.getLogger(this.getClass()).warn("Failed to check if imported table references external java classes : " + ioe.getMessage());
+ }
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(importDir.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(importDir.getBytes(UTF_8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableFateOperation(tableName, AccumuloException.class, FateOperation.TABLE_IMPORT, args, opts);
+ } catch (TableNotFoundException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+
+ }
+
+ @Override
+ public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, exportDir);
+
- List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(Constants.UTF8)), ByteBuffer.wrap(exportDir.getBytes(Constants.UTF8)));
++ List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)), ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
+
+ Map<String,String> opts = Collections.emptyMap();
+
+ try {
+ doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_EXPORT, args, opts);
+ } catch (TableExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public boolean testClassLoad(final String tableName, final String className, final String asTypeName) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName, className, asTypeName);
+
+ try {
+ return ServerClient.executeRaw(instance, new ClientExecReturn<Boolean,ClientService.Client>() {
+ @Override
+ public Boolean execute(ClientService.Client client) throws Exception {
+ return client.checkTableClass(Tracer.traceInfo(), credentials.toThrift(instance), tableName, className, asTypeName);
+ }
+ });
+ } catch (ThriftTableOperationException e) {
+ switch (e.getType()) {
+ case NOTFOUND:
+ throw new TableNotFoundException(e);
+ case NAMESPACE_NOTFOUND:
+ throw new TableNotFoundException(tableName, new NamespaceNotFoundException(e));
+ default:
+ throw new AccumuloException(e.description, e);
+ }
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ }
+ }
+
+ @Override
+ public void attachIterator(String tableName, IteratorSetting setting, EnumSet<IteratorScope> scopes) throws AccumuloSecurityException, AccumuloException,
+ TableNotFoundException {
+ testClassLoad(tableName, setting.getIteratorClass(), SortedKeyValueIterator.class.getName());
+ super.attachIterator(tableName, setting, scopes);
+ }
+
+ @Override
+ public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ testClassLoad(tableName, constraintClassName, Constraint.class.getName());
+ return super.addConstraint(tableName, constraintClassName);
+ }
+
+ private void doTableFateOperation(String tableName, Class<? extends Exception> namespaceNotFoundExceptionClass, FateOperation op, List<ByteBuffer> args,
+ Map<String,String> opts) throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
+ try {
+ doFateOperation(op, args, opts);
+ } catch (NamespaceExistsException e) {
+ // should not happen
+ throw new AssertionError(e);
+ } catch (NamespaceNotFoundException e) {
+ if (namespaceNotFoundExceptionClass == null) {
+ // should not happen
+ throw new AssertionError(e);
+ } else if (AccumuloException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
+ throw new AccumuloException("Cannot create table in non-existent namespace", e);
+ } else if (TableNotFoundException.class.isAssignableFrom(namespaceNotFoundExceptionClass)) {
+ throw new TableNotFoundException(null, tableName, "Namespace not found", e);
+ } else {
+ // should not happen
+ throw new AssertionError(e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
index 32b6e28,6414193..4de95c3
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java
@@@ -16,8 -16,9 +16,10 @@@
*/
package org.apache.accumulo.core.client.impl;
+ import static com.google.common.base.Charsets.UTF_8;
+
import java.security.SecurityPermission;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
@@@ -118,75 -41,36 +120,75 @@@ public class Tables
if (sm != null) {
sm.checkPermission(TABLES_PERMISSION);
}
- return ZooCache.getInstance(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
}
-
+
private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) {
ZooCache zc = getZooCache(instance);
-
+
List<String> tableIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES);
-
TreeMap<String,String> tableMap = new TreeMap<String,String>();
-
+ Map<String,String> namespaceIdToNameMap = new HashMap<String,String>();
+
for (String tableId : tableIds) {
- byte[] tblPath = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
- if (tblPath != null) {
+ byte[] tableName = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME);
+ byte[] nId = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
+ String namespaceName = Namespaces.DEFAULT_NAMESPACE;
+ // create fully qualified table name
+ if (nId == null) {
+ namespaceName = null;
+ } else {
- String namespaceId = new String(nId, Constants.UTF8);
++ String namespaceId = new String(nId, UTF_8);
+ if (!namespaceId.equals(Namespaces.DEFAULT_NAMESPACE_ID)) {
+ try {
+ namespaceName = namespaceIdToNameMap.get(namespaceId);
+ if (namespaceName == null) {
+ namespaceName = Namespaces.getNamespaceName(instance, namespaceId);
+ namespaceIdToNameMap.put(namespaceId, namespaceName);
+ }
+ } catch (NamespaceNotFoundException e) {
+ log.error("Table (" + tableId + ") contains reference to namespace (" + namespaceId + ") that doesn't exist", e);
+ continue;
+ }
+ }
+ }
+ if (tableName != null && namespaceName != null) {
- String tableNameStr = qualified(new String(tableName, Constants.UTF8), namespaceName);
++ String tableNameStr = qualified(new String(tableName, UTF_8), namespaceName);
if (nameAsKey)
- tableMap.put(new String(tblPath, UTF_8), tableId);
+ tableMap.put(tableNameStr, tableId);
else
- tableMap.put(tableId, new String(tblPath, UTF_8));
+ tableMap.put(tableId, tableNameStr);
}
}
-
+
return tableMap;
}
-
+
public static String getTableId(Instance instance, String tableName) throws TableNotFoundException {
+ try {
+ return _getTableId(instance, tableName);
+ } catch (NamespaceNotFoundException e) {
+ throw new TableNotFoundException(tableName, e);
+ }
+ }
+
+ public static String _getTableId(Instance instance, String tableName) throws NamespaceNotFoundException, TableNotFoundException {
String tableId = getNameToIdMap(instance).get(tableName);
- if (tableId == null)
- throw new TableNotFoundException(tableId, tableName, null);
+ if (tableId == null) {
+ // maybe the table exist, but the cache was not updated yet... so try to clear the cache and check again
+ clearCache(instance);
+ tableId = getNameToIdMap(instance).get(tableName);
+ if (tableId == null) {
+ String namespace = qualify(tableName).getFirst();
+ if (Namespaces.getNameToIdMap(instance).containsKey(namespace))
+ throw new TableNotFoundException(null, tableName, null);
+ else
+ throw new NamespaceNotFoundException(null, namespace, null);
+ }
+ }
return tableId;
}
-
+
public static String getTableName(Instance instance, String tableId) throws TableNotFoundException {
String tableName = getIdToNameMap(instance).get(tableId);
if (tableName == null)
@@@ -250,65 -113,11 +252,65 @@@
byte[] state = zc.get(statePath);
if (state == null)
return TableState.UNKNOWN;
-
+
- return TableState.valueOf(new String(state, Constants.UTF8));
+ return TableState.valueOf(new String(state, UTF_8));
}
-
+
public static long getCacheResetCount() {
return cacheResetCount.get();
}
+
+ public static String qualified(String tableName) {
+ return qualified(tableName, Namespaces.DEFAULT_NAMESPACE);
+ }
+
+ public static String qualified(String tableName, String defaultNamespace) {
+ Pair<String,String> qualifiedTableName = Tables.qualify(tableName, defaultNamespace);
+ if (Namespaces.DEFAULT_NAMESPACE.equals(qualifiedTableName.getFirst()))
+ return qualifiedTableName.getSecond();
+ else
+ return qualifiedTableName.toString("", ".", "");
+ }
+
+ public static Pair<String,String> qualify(String tableName) {
+ return qualify(tableName, Namespaces.DEFAULT_NAMESPACE);
+ }
+
+ public static Pair<String,String> qualify(String tableName, String defaultNamespace) {
+ if (!tableName.matches(Tables.VALID_NAME_REGEX))
+ throw new IllegalArgumentException("Invalid table name '" + tableName + "'");
+ if (MetadataTable.OLD_NAME.equals(tableName))
+ tableName = MetadataTable.NAME;
+ if (tableName.contains(".")) {
+ String[] s = tableName.split("\\.", 2);
+ return new Pair<String,String>(s[0], s[1]);
+ }
+ return new Pair<String,String>(defaultNamespace, tableName);
+ }
+
+ /**
+ * Returns the namespace id for a given table ID.
+ *
+ * @param instance
+ * The Accumulo Instance
+ * @param tableId
+ * The tableId
+ * @return The namespace id which this table resides in.
+ * @throws IllegalArgumentException
+ * if the table doesn't exist in ZooKeeper
+ */
+ public static String getNamespaceId(Instance instance, String tableId) throws IllegalArgumentException {
+ ArgumentChecker.notNull(instance, tableId);
+
+ ZooCache zc = getZooCache(instance);
+ byte[] n = zc.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE);
+
+ // We might get null out of ZooCache if this tableID doesn't exist
+ if (null == n) {
+ throw new IllegalArgumentException("Table with id " + tableId + " does not exist");
+ }
+
- return new String(n, Constants.UTF8);
++ return new String(n, UTF_8);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index d58d1e4,a9f5b61..de7bcb2
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@@ -81,10 -82,10 +82,10 @@@ public class Writer
throw new IllegalArgumentException("Can not add empty mutations");
while (true) {
- TabletLocation tabLoc = TabletLocator.getInstance(instance, table).locateTablet(new Text(m.getRow()), false, true, credentials);
+ TabletLocation tabLoc = TabletLocator.getLocator(instance, table).locateTablet(credentials, new Text(m.getRow()), false, true);
if (tabLoc == null) {
- log.trace("No tablet location found for row " + new String(m.getRow(), Constants.UTF8));
+ log.trace("No tablet location found for row " + new String(m.getRow(), UTF_8));
UtilWaitThread.sleep(500);
continue;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
index a071d07,0000000..bf365ef
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/lexicoder/StringLexicoder.java
@@@ -1,40 -1,0 +1,40 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.lexicoder;
+
- import org.apache.accumulo.core.Constants;
++import static com.google.common.base.Charsets.UTF_8;
+
+/**
+ * This lexicoder encodes/decodes a given String to/from bytes without further processing. It can be combined with other encoders like the
+ * {@link ReverseLexicoder} to flip the default sort order.
+ *
+ * @since 1.6.0
+ */
+
+public class StringLexicoder implements Lexicoder<String> {
+
+ @Override
+ public byte[] encode(String data) {
- return data.getBytes(Constants.UTF8);
++ return data.getBytes(UTF_8);
+ }
+
+ @Override
+ public String decode(byte[] data) {
- return new String(data, Constants.UTF8);
++ return new String(data, UTF_8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 500047c,23540f2..563610a
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -27,8 -27,7 +29,7 @@@ import java.util.HashSet
import java.util.List;
import java.util.Set;
- import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.ZooKeeperInstance;
@@@ -203,24 -189,14 +204,24 @@@ public class RangeInputSplit extends In
}
if (in.readBoolean()) {
- String tokenClass = in.readUTF();
- byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
- byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
-
- try {
- token = CredentialHelper.extractToken(tokenClass, tokenBytes);
- } catch (AccumuloSecurityException e) {
- throw new IOException(e);
+ int ordinal = in.readInt();
+ this.tokenSource = TokenSource.values()[ordinal];
+
+ switch (this.tokenSource) {
+ case INLINE:
+ String tokenClass = in.readUTF();
- byte[] base64TokenBytes = in.readUTF().getBytes(Constants.UTF8);
++ byte[] base64TokenBytes = in.readUTF().getBytes(UTF_8);
+ byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
+
+ this.token = AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes);
+ break;
+
+ case FILE:
+ this.tokenFile = in.readUTF();
+
+ break;
+ default:
+ throw new IOException("Cannot parse unknown TokenSource ordinal");
}
}
[47/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 7f2048b,7a1e1ba..c93f680
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index 3543a16,809a7c8..0652cf1
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -16,9 -16,10 +16,9 @@@
*/
package org.apache.accumulo.server.util;
- import java.io.IOException;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ import java.io.IOException;
-
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index f728fec,d40830f..0793c6b
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.HashSet;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 565eaba,43348df..478b3b5
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -16,10 -16,11 +16,10 @@@
*/
package org.apache.accumulo.server.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
- import java.nio.charset.StandardCharsets;
--
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.util.Base64;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f73747d,be85b1b..8550d0b
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Formattable;
import java.util.Formatter;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index 05f28d7,e936b97..01eb477
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@@ -16,8 -16,7 +16,9 @@@
*/
package org.apache.accumulo.server.util;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@@ -70,8 -69,8 +71,8 @@@ public class MasterMetadataUtil
Map<FileRef,Long> bulkLoadedFiles, Credentials credentials, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
Mutation m = extent.getPrevRowUpdateMutation();
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(StandardCharsets.UTF_8)));
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(StandardCharsets.UTF_8)));
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes()));
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes()));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
++ TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
if (lastFlushID > 0)
TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(("" + lastFlushID).getBytes()));
if (lastCompactID > 0)
@@@ -104,7 -103,7 +105,7 @@@
throw new IllegalArgumentException("Metadata entry does not have split ratio (" + metadataEntry + ")");
}
- double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get(), StandardCharsets.UTF_8));
- double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get()));
++ double splitRatio = Double.parseDouble(new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get(), UTF_8));
Value prevEndRowIBW = columns.get(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
@@@ -245,62 -244,41 +246,62 @@@
Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
if (extent.isRootTablet()) {
if (unusedWalLogs != null) {
- IZooReaderWriter zk = ZooReaderWriter.getInstance();
- // unusedWalLogs will contain the location/name of each log in a log set
- // the log set is stored under one of the log names, but not both
- // find the entry under one of the names and delete it.
- String root = MetadataTableUtil.getZookeeperLogLocation();
- boolean foundEntry = false;
- for (String entry : unusedWalLogs) {
- String[] parts = entry.split("/");
- String zpath = root + "/" + parts[parts.length - 1];
- while (true) {
- try {
- if (zk.exists(zpath)) {
- zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
- foundEntry = true;
- }
- break;
- } catch (KeeperException e) {
- log.error(e, e);
- } catch (InterruptedException e) {
- log.error(e, e);
- }
- UtilWaitThread.sleep(1000);
+ updateRootTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId);
+ }
+
+ return;
+ }
+
+ Mutation m = getUpdateForTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId);
+
+ MetadataTableUtil.update(credentials, zooLock, m, extent);
+
+ }
+
+ /**
+ * Update the data file for the root tablet
+ */
+ protected static void updateRootTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time,
+ Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // unusedWalLogs will contain the location/name of each log in a log set
+ // the log set is stored under one of the log names, but not both
+ // find the entry under one of the names and delete it.
+ String root = MetadataTableUtil.getZookeeperLogLocation();
+ boolean foundEntry = false;
+ for (String entry : unusedWalLogs) {
+ String[] parts = entry.split("/");
+ String zpath = root + "/" + parts[parts.length - 1];
+ while (true) {
+ try {
+ if (zk.exists(zpath)) {
+ zk.recursiveDelete(zpath, NodeMissingPolicy.SKIP);
+ foundEntry = true;
}
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
}
- if (unusedWalLogs.size() > 0 && !foundEntry)
- log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
+ UtilWaitThread.sleep(1000);
}
- return;
}
-
+ if (unusedWalLogs.size() > 0 && !foundEntry)
+ log.warn("WALog entry for root tablet did not exist " + unusedWalLogs);
+ }
+
+ /**
+ * Create an update that updates a tablet
+ * @return A Mutation to update a tablet from the given information
+ */
+ protected static Mutation getUpdateForTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time,
+ Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
Mutation m = new Mutation(extent.getMetadataEntry());
-
+
if (dfv.getNumEntries() > 0) {
m.put(DataFileColumnFamily.NAME, path.meta(), new Value(dfv.encode()));
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(StandardCharsets.UTF_8)));
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes()));
++ TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
// stuff in this location
TServerInstance self = getTServerInstance(address, zooLock);
self.putLastLocation(m);
@@@ -313,15 -291,17 +314,15 @@@
m.putDelete(LogColumnFamily.NAME, new Text(entry));
}
}
-
+
for (FileRef scanFile : filesInUseByScans)
- m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value("".getBytes()));
-
+ m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0]));
+
if (mergeFile != null)
m.putDelete(DataFileColumnFamily.NAME, mergeFile.meta());
-
- TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushId + "").getBytes()));
-
- MetadataTableUtil.update(credentials, zooLock, m, extent);
-
+
- TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(StandardCharsets.UTF_8)));
++ TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(UTF_8)));
+
+ return m;
}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index cb97d2a,9cd02eb..8f84169
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index 3bd7b8e,c0d6894..85cc484
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.Property;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index 9f298af,8924c85..d797392
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -16,12 -16,12 +16,12 @@@
*/
package org.apache.accumulo.server.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Map.Entry;
-import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@@ -105,10 -105,9 +105,10 @@@ public class RandomizeVolumes
}
Key key = entry.getKey();
Mutation m = new Mutation(key.getRow());
-
- String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
+
+ String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR
+ + directory;
- m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(StandardCharsets.UTF_8)));
+ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
if (log.isTraceEnabled()) {
log.trace("Replacing " + oldLocation + " with " + newLocation);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 3a8eb0b,a82fa14..b08bf90
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.server.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.FileInputStream;
import java.io.InputStream;
- import java.nio.charset.StandardCharsets;
import java.util.Stack;
import javax.xml.parsers.SAXParser;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index ee01bb2,b1a553d..63f1343
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.util;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index 9543440,401d98c..a376ed6
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.Property;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index a755c0a,6ee1593..5e6542d
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.Property;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index f334613,edbb103..307bb0c
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.List;
import org.apache.accumulo.core.Constants;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index ee20006,7ae4d5a..103aefc
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.watcher;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.zookeeper.ZooUtil;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
index 0f1a005,c5a9528..1452aa7
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
@@@ -16,7 -16,6 +16,8 @@@
*/
package org.apache.accumulo.server.zookeeper;
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@@ -212,14 -200,7 +213,14 @@@ public class DistributedWorkQueue
log.info("Interrupted looking for work", e);
}
}
- }, r.nextInt(60 * 1000), 60 * 1000);
+ }, timerInitialDelay, timerPeriod);
+ }
+
+ /**
- * Adds work to the queue, automatically converting the String to bytes using {@link StandardCharsets#UTF_8}
++ * Adds work to the queue, automatically converting the String to bytes using UTF-8
+ */
+ public void addWork(String workId, String data) throws KeeperException, InterruptedException {
- addWork(workId, data.getBytes(StandardCharsets.UTF_8));
++ addWork(workId, data.getBytes(UTF_8));
}
public void addWork(String workId, byte[] data) throws KeeperException, InterruptedException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
index 9a968a9,1bbae03..e4ef1eb
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.zookeeper;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
index e6d2350,c5374a2..34c9070
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.zookeeper;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index e07c2b9,e15e63e..12bdeb1
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.zookeeper;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index 36ddd3b,3aed7ef..628b981
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@@ -16,7 -16,8 +16,18 @@@
*/
package org.apache.accumulo.server.conf;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
++import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertNull;
++import static org.junit.Assert.assertTrue;
++import static org.easymock.EasyMock.anyObject;
++import static org.easymock.EasyMock.createMock;
++import static org.easymock.EasyMock.eq;
++import static org.easymock.EasyMock.expect;
++import static org.easymock.EasyMock.expectLastCall;
++import static org.easymock.EasyMock.replay;
++import static org.easymock.EasyMock.verify;
+
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@@ -34,16 -35,16 +45,6 @@@ import org.apache.accumulo.fate.zookeep
import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
import org.junit.Before;
import org.junit.Test;
--import static org.junit.Assert.assertEquals;
--import static org.junit.Assert.assertNull;
--import static org.junit.Assert.assertTrue;
--import static org.easymock.EasyMock.anyObject;
--import static org.easymock.EasyMock.createMock;
--import static org.easymock.EasyMock.eq;
--import static org.easymock.EasyMock.expect;
--import static org.easymock.EasyMock.expectLastCall;
--import static org.easymock.EasyMock.replay;
--import static org.easymock.EasyMock.verify;
public class NamespaceConfigurationTest {
private static final String NSID = "namespace";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
index 475afbb,81148a0..cec75c2
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.conf;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.endsWith;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
index 12521f9,2f38e4c..e66ec98
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/TableConfigurationTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.conf;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.eq;
@@@ -84,8 -84,8 +84,8 @@@ public class TableConfigurationTest
@Test
public void testGet_InZK() {
Property p = Property.INSTANCE_SECRET;
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
- .andReturn("sekrit".getBytes(UTF_8));
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(
- "sekrit".getBytes(StandardCharsets.UTF_8));
++ "sekrit".getBytes(UTF_8));
replay(zc);
assertEquals("sekrit", c.get(Property.INSTANCE_SECRET));
}
@@@ -110,10 -110,8 +110,10 @@@
children.add("foo");
children.add("ding");
expect(zc.getChildren(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF)).andReturn(children);
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo")).andReturn("bar".getBytes(UTF_8));
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn("dong".getBytes(UTF_8));
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "foo"))
- .andReturn("bar".getBytes(StandardCharsets.UTF_8));
++ .andReturn("bar".getBytes(UTF_8));
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + "ding")).andReturn(
- "dong".getBytes(StandardCharsets.UTF_8));
++ "dong".getBytes(UTF_8));
replay(zc);
c.getProperties(props, filter);
assertEquals(2, props.size());
@@@ -137,8 -135,8 +137,8 @@@
public void testInvalidateCache() {
// need to do a get so the accessor is created
Property p = Property.INSTANCE_SECRET;
- expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey()))
- .andReturn("sekrit".getBytes(UTF_8));
+ expect(zc.get(ZooUtil.getRoot(iid) + Constants.ZTABLES + "/" + TID + Constants.ZTABLE_CONF + "/" + p.getKey())).andReturn(
- "sekrit".getBytes(StandardCharsets.UTF_8));
++ "sekrit".getBytes(UTF_8));
zc.clear();
replay(zc);
c.get(Property.INSTANCE_SECRET);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
index 742ccdd,4369119..210d240
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessorTest.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.conf;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index e5f3955,83e83cf..e3fc69d
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.master;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@@ -301,53 -291,20 +302,53 @@@ public class Master implements LiveTSer
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
final String zooRoot = ZooUtil.getRoot(instance);
- if (accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO) {
- log.debug("Handling updates for version " + ServerConstants.TWO_DATA_VERSIONS_AGO);
+ log.debug("Handling updates for version " + accumuloPersistentVersion);
- log.debug("Cleaning out remnants of logger role.");
- zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
- zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
+ log.debug("Cleaning out remnants of logger role.");
+ zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP);
- final byte[] zero = new byte[] {'0'};
- log.debug("Initializing recovery area.");
- zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
+ final byte[] zero = new byte[] {'0'};
+ log.debug("Initializing recovery area.");
+ zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP);
- for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
- log.debug("Prepping table " + id + " for compaction cancellations.");
- zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
+ for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
+ log.debug("Prepping table " + id + " for compaction cancellations.");
+ zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
+ }
+
+ @SuppressWarnings("deprecation")
+ String zpath = zooRoot + Constants.ZCONFIG + "/" + Property.TSERV_WAL_SYNC_METHOD.getKey();
+ // is the entire instance set to use flushing vs sync?
+ boolean flushDefault = false;
+ try {
+ byte data[] = zoo.getData(zpath, null);
- if (new String(data, StandardCharsets.UTF_8).endsWith("flush")) {
++ if (new String(data, UTF_8).endsWith("flush")) {
+ flushDefault = true;
+ }
+ } catch (KeeperException.NoNodeException ex) {
+ // skip
+ }
+ for (String id : zoo.getChildren(zooRoot + Constants.ZTABLES)) {
+ log.debug("Converting table " + id + " WALog setting to Durability");
+ try {
+ @SuppressWarnings("deprecation")
+ String path = zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/" + Property.TABLE_WALOG_ENABLED.getKey();
+ byte[] data = zoo.getData(path, null);
- boolean useWAL = Boolean.parseBoolean(new String(data, StandardCharsets.UTF_8));
++ boolean useWAL = Boolean.parseBoolean(new String(data, UTF_8));
+ zoo.recursiveDelete(path, NodeMissingPolicy.FAIL);
+ path = zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_CONF + "/" + Property.TABLE_DURABILITY.getKey();
+ if (useWAL) {
+ if (flushDefault) {
+ zoo.putPersistentData(path, "flush".getBytes(), NodeExistsPolicy.SKIP);
+ } else {
+ zoo.putPersistentData(path, "sync".getBytes(), NodeExistsPolicy.SKIP);
+ }
+ } else {
+ zoo.putPersistentData(path, "none".getBytes(), NodeExistsPolicy.SKIP);
+ }
+ } catch (KeeperException.NoNodeException ex) {
+ // skip it
}
}
@@@ -385,8 -342,8 +386,8 @@@
// rename metadata table
log.debug("Upgrade renaming table " + MetadataTable.OLD_NAME + " (ID: " + MetadataTable.ID + ") to " + MetadataTable.NAME);
- zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME, Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8),
- NodeExistsPolicy.OVERWRITE);
+ zoo.putPersistentData(tables + "/" + MetadataTable.ID + Constants.ZTABLE_NAME,
- Tables.qualify(MetadataTable.NAME).getSecond().getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
++ Tables.qualify(MetadataTable.NAME).getSecond().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
moveRootTabletToRootTable(zoo);
@@@ -1087,41 -1030,6 +1088,41 @@@
while (!clientService.isServing()) {
UtilWaitThread.sleep(100);
}
+
+ // Start the daemon to scan the replication table and make units of work
+ replicationWorkDriver = new ReplicationDriver(this);
+ replicationWorkDriver.start();
+
+ // Start the daemon to assign work to tservers to replicate to our peers
+ try {
+ replicationWorkAssigner = new WorkDriver(this, getConnector());
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ log.error("Caught exception trying to initialize replication WorkDriver", e);
+ throw new RuntimeException(e);
+ }
+ replicationWorkAssigner.start();
+
+ // Start the replication coordinator which assigns tservers to service replication requests
+ ReplicationCoordinator.Processor<ReplicationCoordinator.Iface> replicationCoordinatorProcessor = new ReplicationCoordinator.Processor<ReplicationCoordinator.Iface>(
+ RpcWrapper.service(new MasterReplicationCoordinator(this)));
+ ServerAddress replAddress = TServerUtils.startServer(getConfiguration(), hostname, Property.MASTER_REPLICATION_COORDINATOR_PORT,
+ replicationCoordinatorProcessor, "Master Replication Coordinator", "Replication Coordinator", null, Property.MASTER_REPLICATION_COORDINATOR_MINTHREADS,
+ Property.MASTER_REPLICATION_COORDINATOR_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
+
+ log.info("Started replication coordinator service at " + replAddress.address);
+
+ // Advertise that port we used so peers don't have to be told what it is
+ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_REPLICATION_COORDINATOR_ADDR,
- replAddress.address.toString().getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
++ replAddress.address.toString().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ final SystemCredentials creds = SystemCredentials.get();
+ try {
+ ReplicationMetrics beanImpl = new ReplicationMetrics(this.instance.getConnector(creds.getPrincipal(), creds.getToken()));
+ beanImpl.register();
+ } catch (Exception e) {
+ log.error("Error registering Replication metrics with JMX", e);
+ }
+
while (clientService.isServing()) {
UtilWaitThread.sleep(500);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
index ba7a52a,5e478cb..c636dbb
--- a/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/recovery/RecoveryManager.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.master.recovery;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.FileNotFoundException;
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@@ -114,10 -114,9 +115,10 @@@ public class RecoveryManager
}
- private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
+ private void initiateSort(String sortId, String source, final String destination, AccumuloConfiguration aconf)
+ throws KeeperException, InterruptedException, IOException {
String work = source + "|" + destination;
- new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY, aconf).addWork(sortId, work.getBytes(StandardCharsets.UTF_8));
- new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(UTF_8));
++ new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY, aconf).addWork(sortId, work.getBytes(UTF_8));
synchronized (this) {
sortsQueued.add(sortId);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
index 795fabb,0000000..8218c8a
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
@@@ -1,107 -1,0 +1,108 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.util.Iterator;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.replication.ReplicationConstants;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinatorErrorCode;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinatorException;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.thrift.TException;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Choose a tserver to service a replication task
+ */
+public class MasterReplicationCoordinator implements ReplicationCoordinator.Iface {
+ private static final Logger log = LoggerFactory.getLogger(MasterReplicationCoordinator.class);
+
+ private final Master master;
+ private final Instance inst;
+ private final Random rand;
+ private final ZooReader reader;
+ private final SecurityOperation security;
+
+ public MasterReplicationCoordinator(Master master) {
+ this(master, new ZooReader(master.getInstance().getZooKeepers(), master.getInstance().getZooKeepersSessionTimeOut()));
+ }
+
+ protected MasterReplicationCoordinator(Master master, ZooReader reader) {
+ this.master = master;
+ this.rand = new Random(358923462l);
+ this.inst = master.getInstance();
+ this.reader = reader;
+ this.security = SecurityOperation.getInstance(inst.getInstanceID(), false);
+ }
+
+
+ @Override
+ public String getServicerAddress(String remoteTableId, TCredentials creds) throws ReplicationCoordinatorException, TException {
+ try {
+ security.authenticateUser(SystemCredentials.get().toThrift(inst), creds);
+ } catch (ThriftSecurityException e) {
+ log.error("{} failed to authenticate for replication to {}", creds.getPrincipal(), remoteTableId);
+ throw new ReplicationCoordinatorException(ReplicationCoordinatorErrorCode.CANNOT_AUTHENTICATE, "Could not authenticate " + creds.getPrincipal());
+ }
+
+ Set<TServerInstance> tservers = master.onlineTabletServers();
+ if (tservers.isEmpty()) {
+ throw new ReplicationCoordinatorException(ReplicationCoordinatorErrorCode.NO_AVAILABLE_SERVERS, "No tservers are available for replication");
+ }
+
+ TServerInstance tserver = getRandomTServer(tservers, rand.nextInt(tservers.size()));
+ String replServiceAddr;
+ try {
- replServiceAddr = new String(reader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver.hostPort(), null), StandardCharsets.UTF_8);
++ replServiceAddr = new String(reader.getData(ZooUtil.getRoot(inst) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver.hostPort(), null), UTF_8);
+ } catch (KeeperException | InterruptedException e) {
+ log.error("Could not fetch repliation service port for tserver", e);
+ throw new ReplicationCoordinatorException(ReplicationCoordinatorErrorCode.SERVICE_CONFIGURATION_UNAVAILABLE,
+ "Could not determine port for replication service running at " + tserver.hostPort());
+ }
+
+ return replServiceAddr;
+ }
+
+ protected TServerInstance getRandomTServer(Set<TServerInstance> tservers, int offset) {
+ Preconditions.checkArgument(tservers.size() > offset, "Must provide an offset less than the size of the set");
+ Iterator<TServerInstance> iter = tservers.iterator();
+ while (offset > 0) {
+ iter.next();
+ offset--;
+ }
+
+ return iter.next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
index 3da225d,a674557..2bbbba3
--- a/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/state/SetGoalState.java
@@@ -16,10 -16,9 +16,10 @@@
*/
package org.apache.accumulo.master.state;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
import org.apache.accumulo.core.security.SecurityUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
index b55b315,10cf224..ce25310
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/BulkImport.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.master.tableOps;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
index eb49a35,fd2441c..374fc24
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CancelCompactions.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.master.tableOps;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index 2673f03,fbaa70d..13ef68e
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.master.tableOps;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index 5d032d1,037455c..9c397db
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.master.tableOps;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.DataOutputStream;
@@@ -258,10 -259,9 +259,10 @@@ class WriteExportFiles extends MasterRe
Map<String,String> siteConfig = conn.instanceOperations().getSiteConfiguration();
Map<String,String> systemConfig = conn.instanceOperations().getSystemConfiguration();
- TableConfiguration tableConfig = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableID);
+ ServerConfigurationFactory factory = new ServerConfigurationFactory(conn.getInstance());
+ TableConfiguration tableConfig = factory.getTableConfiguration(tableID);
- OutputStreamWriter osw = new OutputStreamWriter(dataOut, StandardCharsets.UTF_8);
+ OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
// only put props that are different than defaults and higher level configurations
zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 44ca898,05ec263..35067ce
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.master.tableOps;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index 7965617,4f381ef..5d511ac
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.master.tableOps;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 6961776,3e8704e..64947fb
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.master.tableOps;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.math.BigInteger;
- import java.nio.charset.StandardCharsets;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
index aeb07e2,a9058b7..53c7c6f
--- a/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
@@@ -16,7 -16,7 +16,7 @@@
*/
package org.apache.accumulo.master.tserverOps;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 6135fb0,75d1e9e..49bb56d
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.monitor;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.net.InetAddress;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index b0c20ec,78dd95e..7aefa1d
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.monitor;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collection;
import java.util.SortedSet;
import java.util.TreeSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index b631320,7812603..2feb804
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.monitor.servlets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 2395335,bb7e690..3347647
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.monitor.servlets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.io.InputStream;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index bf38064,4975428..6b2b31f
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.monitor.servlets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
index b639226,3cd635e..405d6df
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/XMLServlet.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.monitor.servlets;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index c1531cb,1c464d4..475cae1
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.monitor.servlets.trace;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
----------------------------------------------------------------------
diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index e7e046e,6094911..4858b8a
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.tracer;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.channels.ServerSocketChannel;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index ff0097a,4d9a6c2..f7bda49
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@@ -16,11 -16,12 +16,11 @@@
*/
package org.apache.accumulo.tserver;
- import java.io.IOException;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ import java.io.IOException;
-
+import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.trace.TraceFileSystem;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 29ed8d7,94be0bb..54c75f8
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -16,19 -16,22 +16,19 @@@
*/
package org.apache.accumulo.tserver;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
-import java.net.Socket;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@@ -412,8 -1198,8 +412,8 @@@ public class TabletServer implements Ru
List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
- String tableId = new String(textent.getTable(), StandardCharsets.UTF_8);
+ String tableId = new String(textent.getTable(), UTF_8);
- if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(getInstance(), tableId), range, columns, ssiList, ssio, authorizations))
throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
if (!security.userHasAuthorizations(credentials, authorizations))
@@@ -552,9 -1347,9 +552,9 @@@
public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
// find all of the tables that need to be scanned
- HashSet<String> tables = new HashSet<String>();
+ final HashSet<String> tables = new HashSet<String>();
for (TKeyExtent keyExtent : tbatch.keySet()) {
- tables.add(new String(keyExtent.getTable(), StandardCharsets.UTF_8));
+ tables.add(new String(keyExtent.getTable(), UTF_8));
}
if (tables.size() != 1)
@@@ -950,14 -1755,14 +950,14 @@@
}
@Override
- public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation, TDurability tdurability) throws NotServingTabletException,
ConstraintViolationException, ThriftSecurityException {
- final String tableId = new String(tkeyExtent.getTable(), StandardCharsets.UTF_8);
- String tableId = new String(tkeyExtent.getTable(), UTF_8);
- if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
++ final String tableId = new String(tkeyExtent.getTable(), UTF_8);
+ if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(getInstance(), tableId)))
throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
- KeyExtent keyExtent = new KeyExtent(tkeyExtent);
- Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ final KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ final Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
if (tablet == null) {
throw new NotServingTabletException(tkeyExtent);
}
@@@ -2260,30 -3106,7 +2260,30 @@@
return address;
}
- ZooLock getLock() {
+ private HostAndPort startReplicationService() throws UnknownHostException {
+ ReplicationServicer.Iface repl = RpcWrapper.service(new ReplicationServicerHandler(HdfsZooInstance.getInstance()));
+ ReplicationServicer.Processor<ReplicationServicer.Iface> processor = new ReplicationServicer.Processor<ReplicationServicer.Iface>(repl);
+ AccumuloConfiguration conf = getConfiguration();
+ Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null ? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
+ ServerAddress sp = TServerUtils.startServer(conf, clientAddress.getHostText(), Property.REPLICATION_RECEIPT_SERVICE_PORT, processor,
+ "ReplicationServicerHandler", "Replication Servicer", null, Property.REPLICATION_MIN_THREADS, Property.REPLICATION_THREADCHECK, maxMessageSizeProperty);
+ this.replServer = sp.server;
+ log.info("Started replication service on " + sp.address);
+
+ try {
+ // The replication service is unique to the thrift service for a tserver, not just a host.
+ // Advertise the host and port for replication service given the host and port for the tserver.
+ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(getInstance()) + ReplicationConstants.ZOO_TSERVERS + "/" + clientAddress.toString(),
- sp.address.toString().getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
++ sp.address.toString().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ } catch (Exception e) {
+ log.error("Could not advertise replication service port", e);
+ throw new RuntimeException(e);
+ }
+
+ return sp.address;
+ }
+
+ public ZooLock getLock() {
return tabletServerLock;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 0ea455c,7158ea8..b7d5b0e
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@@ -16,14 -16,21 +16,20 @@@
*/
package org.apache.accumulo.tserver.log;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+ import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_FINISH;
+ import static org.apache.accumulo.tserver.logger.LogEvents.COMPACTION_START;
+ import static org.apache.accumulo.tserver.logger.LogEvents.DEFINE_TABLET;
+ import static org.apache.accumulo.tserver.logger.LogEvents.MANY_MUTATIONS;
+ import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
+
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.channels.ClosedChannelException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
index 1468fec,2c807fd..81cd593
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileValue.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.tserver.logger;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index a444b45,29eefc8..df2f831
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.tserver.logger;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
index fbae6f2,0000000..4b57813
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
@@@ -1,181 -1,0 +1,182 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicaSystemHelper;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.replication.DistributedWorkQueueWorkAssignerHelper;
+import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Transmit the given data to a peer
+ */
+public class ReplicationProcessor implements Processor {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationProcessor.class);
+
+ private final Instance inst;
+ private final AccumuloConfiguration conf;
+ private final VolumeManager fs;
+ private final Credentials creds;
+ private final ReplicaSystemHelper helper;
+
+ public ReplicationProcessor(Instance inst, AccumuloConfiguration conf, VolumeManager fs, Credentials creds) {
+ this.inst = inst;
+ this.conf = conf;
+ this.fs = fs;
+ this.creds = creds;
+ this.helper = new ReplicaSystemHelper(inst, creds);
+ }
+
+ @Override
+ public ReplicationProcessor newProcessor() {
+ return new ReplicationProcessor(inst, new ServerConfigurationFactory(inst).getConfiguration(), fs, creds);
+ }
+
+ @Override
+ public void process(String workID, byte[] data) {
+ ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID).getValue();
- String file = new String(data, StandardCharsets.UTF_8);
++ String file = new String(data, UTF_8);
+
+ log.debug("Received replication work for {} to {}", file, target);
+
+ ReplicaSystem replica;
+ try {
+ replica = getReplicaSystem(target);
+ } catch (Exception e) {
+ log.error("Could not instantiate ReplicaSystem for {}, waiting before returning the work", target, e);
+ try {
+ // TODO configurable
+ Thread.sleep(5000);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+
+ return;
+ }
+
+ Status status;
+ try {
+ status = getStatus(file, target);
+ } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
+ log.error("Could not look for replication record", e);
+ throw new IllegalStateException("Could not look for replication record", e);
+ } catch (InvalidProtocolBufferException e) {
+ log.error("Could not deserialize Status from Work section for {} and ", file, target);
+ throw new RuntimeException("Could not parse Status for work record", e);
+ } catch (NoSuchElementException e) {
+ log.error("Assigned work for {} to {} but could not find work record", file, target);
+ return;
+ }
+
+ log.debug("Current status for {} replicating to {}: {}", file, target, ProtobufUtil.toString(status));
+
+ // We don't need to do anything (shouldn't have gotten this work record in the first place)
+ if (!StatusUtil.isWorkRequired(status)) {
+ log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
+ return;
+ }
+
+ // Sanity check that nothing bad happened and our replication source still exists
+ Path filePath = new Path(file);
+ try {
+ if (!doesFileExist(filePath, target)) {
+ return;
+ }
+ } catch (IOException e) {
+ log.error("Could not determine if file exists {}", filePath, e);
+ throw new RuntimeException(e);
+ }
+
+ log.debug("Replicating {} to {} using {}", filePath, target, replica.getClass().getName());
+
+ replica.replicate(filePath, status, target, getHelper());
+ }
+
+ protected ReplicaSystemHelper getHelper() {
+ return helper;
+ }
+
+ protected ReplicaSystem getReplicaSystem(ReplicationTarget target) {
+ // Find the configured replication peer so we know how to replicate to it
+ // Classname,Configuration
+ String peerType = getPeerType(target.getPeerName());
+
+ // Get the peer that we're replicating to
+ return ReplicaSystemFactory.get(peerType);
+ }
+
+ protected String getPeerType(String peerName) {
+ // Find the configured replication peer so we know how to replicate to it
+ Map<String,String> configuredPeers = conf.getAllPropertiesWithPrefix(Property.REPLICATION_PEERS);
+ String peerType = configuredPeers.get(Property.REPLICATION_PEERS.getKey() + peerName);
+ if (null == peerType) {
+ String msg = "Cannot process replication for unknown peer: " + peerName;
+ log.warn(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ return peerType;
+ }
+
+ protected boolean doesFileExist(Path filePath, ReplicationTarget target) throws IOException {
+ if (!fs.exists(filePath)) {
+ log.warn("Received work request for {} and {}, but the file doesn't exist", filePath, target);
+ return false;
+ }
+
+ return true;
+ }
+
+ protected Status getStatus(String file, ReplicationTarget target) throws TableNotFoundException, AccumuloException, AccumuloSecurityException,
+ InvalidProtocolBufferException {
+ Scanner s = ReplicationTable.getScanner(inst.getConnector(creds.getPrincipal(), creds.getToken()));
+ s.setRange(Range.exact(file));
+ s.fetchColumn(WorkSection.NAME, target.toText());
+
+ return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get());
+ }
+}
[46/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index d82827e,0000000..95c4f07
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1,2675 -1,0 +1,2676 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Durability;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.impl.DurabilityImpl;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationConfigurationUtil;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.tableOps.CompactionIterators;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.FileUtil;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.ReplicationTableUtil;
+import org.apache.accumulo.server.util.TabletOperations;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.tserver.InMemoryMap;
+import org.apache.accumulo.tserver.MinorCompactionReason;
+import org.apache.accumulo.tserver.TConstraintViolationException;
+import org.apache.accumulo.tserver.TLevel;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.TooManyFilesException;
+import org.apache.accumulo.tserver.TservConstraintEnv;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.apache.accumulo.tserver.compaction.WriteParameters;
+import org.apache.accumulo.tserver.constraints.ConstraintChecker;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.tserver.tablet.Compactor.CompactionCanceledException;
+import org.apache.accumulo.tserver.tablet.Compactor.CompactionEnv;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ *
+ * Provide access to a single row range in a living TabletServer.
+ *
+ */
+public class Tablet implements TabletCommitter {
+ static private final Logger log = Logger.getLogger(Tablet.class);
+ static private final List<LogEntry> NO_LOG_ENTRIES = Collections.emptyList();
+
+ private final TabletServer tabletServer;
+ private final KeyExtent extent;
+ private final TabletResourceManager tabletResources;
+ private final DatafileManager datafileManager;
+ private final TableConfiguration tableConfiguration;
+ private final String tabletDirectory;
+ private final Path location; // absolute path of this tablets dir
+
+ private final TabletMemory tabletMemory;
+
+ private final TabletTime tabletTime;
+ private final Object timeLock = new Object();
+ private long persistedTime;
+
+ private TServerInstance lastLocation;
+ private volatile boolean tableDirChecked = false;
+
+ private final AtomicLong dataSourceDeletions = new AtomicLong(0);
+
+ public long getDataSourceDeletions() {
+ return dataSourceDeletions.get();
+ }
+
+ private final Set<ScanDataSource> activeScans = new HashSet<ScanDataSource>();
+
+ private static enum CloseState {
+ OPEN, CLOSING, CLOSED, COMPLETE
+ }
+
+ private volatile CloseState closeState = CloseState.OPEN;
+
+ private boolean updatingFlushID = false;
+
+ private long lastFlushID = -1;
+ private long lastCompactID = -1;
+
+ private static class CompactionWaitInfo {
+ long flushID = -1;
+ long compactionID = -1;
+ }
+
+ // stores info about user initiated major compaction that is waiting on a minor compaction to finish
+ private CompactionWaitInfo compactionWaitInfo = new CompactionWaitInfo();
+
+ static enum CompactionState {
+ WAITING_TO_START, IN_PROGRESS
+ };
+
+ private volatile CompactionState minorCompactionState = null;
+ private volatile CompactionState majorCompactionState = null;
+
+ private final Set<MajorCompactionReason> majorCompactionQueued = Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
+
+ private final AtomicReference<ConstraintChecker> constraintChecker = new AtomicReference<ConstraintChecker>();
+
+ private int writesInProgress = 0;
+
+ private final TabletStatsKeeper timer = new TabletStatsKeeper();
+
+ private final Rate queryRate = new Rate(0.2);
+ private long queryCount = 0;
+
+ private final Rate queryByteRate = new Rate(0.2);
+ private long queryBytes = 0;
+
+ private final Rate ingestRate = new Rate(0.2);
+ private long ingestCount = 0;
+
+ private final Rate ingestByteRate = new Rate(0.2);
+ private long ingestBytes = 0;
+
+ private byte[] defaultSecurityLabel = new byte[0];
+
+ private long lastMinorCompactionFinishTime = 0;
+ private long lastMapFileImportTime = 0;
+
+ private volatile long numEntries = 0;
+ private volatile long numEntriesInMemory = 0;
+
+ private final Rate scannedRate = new Rate(0.2);
+ private final AtomicLong scannedCount = new AtomicLong(0);
+
+ private final ConfigurationObserver configObserver;
+
+ private final int logId;
+
+ @Override
+ public int getLogId() {
+ return logId;
+ }
+
+ public static class LookupResult {
+ public List<Range> unfinishedRanges = new ArrayList<Range>();
+ public long bytesAdded = 0;
+ public long dataSize = 0;
+ public boolean closed = false;
+ }
+
+ FileRef getNextMapFilename(String prefix) throws IOException {
+ String extension = FileOperations.getNewFileExtension(tableConfiguration);
+ checkTabletDir();
+ return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension);
+ }
+
+ private void checkTabletDir() throws IOException {
+ if (!tableDirChecked) {
+ FileStatus[] files = null;
+ try {
+ files = getTabletServer().getFileSystem().listStatus(location);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+
+ if (files == null) {
+ if (location.getName().startsWith(Constants.CLONE_PREFIX))
+ log.debug("Tablet " + extent + " had no dir, creating " + location); // its a clone dir...
+ else
+ log.warn("Tablet " + extent + " had no dir, creating " + location);
+
+ getTabletServer().getFileSystem().mkdirs(location);
+ }
+ tableDirChecked = true;
+ }
+ }
+
+ /**
+ * Only visible for testing
+ */
+ @VisibleForTesting
+ protected Tablet(TabletTime tabletTime, String tabletDirectory, int logId, Path location, DatafileManager datafileManager, TabletServer tabletServer,
+ TabletResourceManager tabletResources, TabletMemory tabletMemory, TableConfiguration tableConfiguration, KeyExtent extent,
+ ConfigurationObserver configObserver) {
+ this.tabletTime = tabletTime;
+ this.tabletDirectory = tabletDirectory;
+ this.logId = logId;
+ this.location = location;
+ this.datafileManager = datafileManager;
+ this.tabletServer = tabletServer;
+ this.tabletResources = tabletResources;
+ this.tabletMemory = tabletMemory;
+ this.tableConfiguration = tableConfiguration;
+ this.extent = extent;
+ this.configObserver = configObserver;
+ }
+
+ public Tablet(TabletServer tabletServer, KeyExtent extent, TabletResourceManager trm, SplitInfo info) throws IOException {
+ this(tabletServer, new Text(info.getDir()), extent, trm, info.getDatafiles(), info.getTime(), info.getInitFlushID(), info.getInitCompactID(), info
+ .getLastLocation());
+ splitCreationTime = System.currentTimeMillis();
+ }
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<FileRef,DataFileValue> datafiles,
+ String time, long initFlushID, long initCompactID, TServerInstance lastLocation) throws IOException {
+ this(tabletServer, extent, location, trm, NO_LOG_ENTRIES, datafiles, time, lastLocation, new HashSet<FileRef>(), initFlushID, initCompactID);
+ }
+
+ private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ SortedMap<Key,Value> entries;
+
+ if (extent.isRootTablet()) {
+ return null;
+ } else {
+ entries = new TreeMap<Key,Value>();
+ Text rowName = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
+ entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
+ }
+ }
+ }
+
+ if (entries.size() == 1)
+ return entries.values().iterator().next().toString();
+ return null;
+ }
+
+ private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+
+ TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
+
+ if (extent.isRootTablet()) { // the meta0 tablet
+ Path location = new Path(MetadataTableUtil.getRootTabletDir());
+
+ // cleanUpFiles() has special handling for delete. files
+ FileStatus[] files = fs.listStatus(location);
+ Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
+ for (String good : goodPaths) {
+ Path path = new Path(good);
+ String filename = path.getName();
+ FileRef ref = new FileRef(location.toString() + "/" + filename, path);
+ DataFileValue dfv = new DataFileValue(0, 0);
+ datafiles.put(ref, dfv);
+ }
+ } else {
+
+ Text rowName = extent.getMetadataEntry();
+
+ String tableId = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
+ ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), tableId, Authorizations.EMPTY);
+
+ // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
+ // reduced batch size to improve performance
+ // changed here after endKeys were implemented from 10 to 1000
+ mdScanner.setBatchSize(1000);
+
+ // leave these in, again, now using endKey for safety
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+
+ mdScanner.setRange(new Range(rowName));
+
+ for (Entry<Key,Value> entry : mdScanner) {
+ if (entry.getKey().compareRow(rowName) != 0) {
+ break;
+ }
+
+ FileRef ref = new FileRef(fs, entry.getKey());
+ datafiles.put(ref, new DataFileValue(entry.getValue().get()));
+ }
+ }
+ return datafiles;
+ }
+
+ private static List<LogEntry> lookupLogEntries(KeyExtent ke, SortedMap<Key,Value> tabletsKeyValues) {
+ List<LogEntry> logEntries = new ArrayList<LogEntry>();
+
+ if (ke.isMeta()) {
+ try {
+ logEntries = MetadataTableUtil.getLogEntries(SystemCredentials.get(), ke);
+ } catch (Exception ex) {
+ throw new RuntimeException("Unable to read tablet log entries", ex);
+ }
+ } else {
+ log.debug("Looking at metadata " + tabletsKeyValues);
+ Text row = ke.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row)) {
+ if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
+ logEntries.add(LogEntry.fromKeyValue(key, entry.getValue()));
+ }
+ }
+ }
+ }
+
+ log.debug("got " + logEntries + " for logs for " + ke);
+ return logEntries;
+ }
+
+ private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
+ HashSet<FileRef> scanFiles = new HashSet<FileRef>();
+
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
+ scanFiles.add(new FileRef(fs, key));
+ }
+ }
+
+ return scanFiles;
+ }
+
+ private static long lookupFlushID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ return Long.parseLong(entry.getValue().toString());
+ }
+
+ return -1;
+ }
+
+ private static long lookupCompactID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ return Long.parseLong(entry.getValue().toString());
+ }
+
+ return -1;
+ }
+
+ private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
+ return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
+ }
+ }
+ return null;
+ }
+
+ public Tablet(TabletServer tabletServer, KeyExtent extent, Text location, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
+ throws IOException {
+ this(tabletServer, extent, location, trm, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getConfiguration(),
+ tabletServer.getFileSystem(), extent, tabletsKeyValues), lookupTime(tabletServer.getConfiguration(), extent, tabletsKeyValues), lookupLastServer(
+ extent, tabletsKeyValues), lookupScanFiles(extent, tabletsKeyValues, tabletServer.getFileSystem()), lookupFlushID(extent, tabletsKeyValues),
+ lookupCompactID(extent, tabletsKeyValues));
+ }
+
+ /**
+ * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
+ */
+ private Tablet(final TabletServer tabletServer, final KeyExtent extent, final Text location, final TabletResourceManager trm,
+ final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time, final TServerInstance lastLocation,
+ Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
+
+ TableConfiguration tblConf = tabletServer.getTableConfiguration(extent);
+ if (null == tblConf) {
+ Tables.clearCache(tabletServer.getInstance());
+ tblConf = tabletServer.getTableConfiguration(extent);
+ if (null == tblConf) {
+ // Not guaranteed to be non-null, but should be. A failed load will be re-assigned though..
+ log.warn("Could not get table configuration for " + extent.getTableId().toString());
+ }
+ }
+
+ this.tableConfiguration = tblConf;
+
+ TabletFiles tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getLock(), tabletServer.getFileSystem(), extent, new TabletFiles(location.toString(),
+ rawLogEntries, rawDatafiles), ReplicationConfigurationUtil.isEnabled(extent, this.tableConfiguration));
+
+ Path locationPath;
+
+ if (tabletPaths.dir.contains(":")) {
+ locationPath = new Path(tabletPaths.dir.toString());
+ } else {
+ locationPath = tabletServer.getFileSystem().getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString());
+ }
+
+ final List<LogEntry> logEntries = tabletPaths.logEntries;
+ final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
+
+ this.location = locationPath;
+ this.lastLocation = lastLocation;
+ this.tabletDirectory = tabletPaths.dir;
+
+ this.extent = extent;
+ this.tabletResources = trm;
+
+ this.lastFlushID = initFlushID;
+ this.lastCompactID = initCompactID;
+
+ if (extent.isRootTablet()) {
+ long rtime = Long.MIN_VALUE;
+ for (FileRef ref : datafiles.keySet()) {
+ Path path = ref.path();
+ FileSystem ns = tabletServer.getFileSystem().getVolumeByPath(path).getFileSystem();
+ FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
+ long maxTime = -1;
+ try {
+
+ while (reader.hasTop()) {
+ maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
+ reader.next();
+ }
+
+ } finally {
+ reader.close();
+ }
+
+ if (maxTime > rtime) {
+ time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
+ rtime = maxTime;
+ }
+ }
+ }
+ if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
+ // recovery... old root tablet has no data, so time doesn't matter:
+ time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+ }
+
+ this.tabletServer = tabletServer;
+ this.logId = tabletServer.createLogId(extent);
+
+ setupDefaultSecurityLabels(extent);
+
+ tabletMemory = new TabletMemory(this);
+ tabletTime = TabletTime.getInstance(time);
+ persistedTime = tabletTime.getTime();
+
+ tableConfiguration.addObserver(configObserver = new ConfigurationObserver() {
+
+ private void reloadConstraints() {
+ constraintChecker.set(new ConstraintChecker(tableConfiguration));
+ }
+
+ @Override
+ public void propertiesChanged() {
+ reloadConstraints();
+
+ try {
+ setupDefaultSecurityLabels(extent);
+ } catch (Exception e) {
+ log.error("Failed to reload default security labels for extent: " + extent.toString());
+ }
+ }
+
+ @Override
+ public void propertyChanged(String prop) {
+ if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
+ reloadConstraints();
+ else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
+ try {
+ log.info("Default security labels changed for extent: " + extent.toString());
+ setupDefaultSecurityLabels(extent);
+ } catch (Exception e) {
+ log.error("Failed to reload default security labels for extent: " + extent.toString());
+ }
+ }
+
+ }
+
+ @Override
+ public void sessionExpired() {
+ log.debug("Session expired, no longer updating per table props...");
+ }
+
+ });
+
+ tableConfiguration.getNamespaceConfiguration().addObserver(configObserver);
+
+ // Force a load of any per-table properties
+ configObserver.propertiesChanged();
+
+ if (!logEntries.isEmpty()) {
+ log.info("Starting Write-Ahead Log recovery for " + this.extent);
+ // count[0] = entries used on tablet
+ // count[1] = track max time from walog entries wihtout timestamps
+ final long[] count = new long[2];
+ final CommitSession commitSession = getTabletMemory().getCommitSession();
+ count[1] = Long.MIN_VALUE;
+ try {
+ Set<String> absPaths = new HashSet<String>();
+ for (FileRef ref : datafiles.keySet())
+ absPaths.add(ref.path().toString());
+
+ tabletServer.recover(this.getTabletServer().getFileSystem(), extent, tableConfiguration, logEntries, absPaths, new MutationReceiver() {
+ @Override
+ public void receive(Mutation m) {
+ // LogReader.printMutation(m);
+ Collection<ColumnUpdate> muts = m.getUpdates();
+ for (ColumnUpdate columnUpdate : muts) {
+ if (!columnUpdate.hasTimestamp()) {
+ // if it is not a user set timestamp, it must have been set
+ // by the system
+ count[1] = Math.max(count[1], columnUpdate.getTimestamp());
+ }
+ }
+ getTabletMemory().mutate(commitSession, Collections.singletonList(m));
+ count[0]++;
+ }
+ });
+
+ if (count[1] != Long.MIN_VALUE) {
+ tabletTime.useMaxTimeFromWALog(count[1]);
+ }
+ commitSession.updateMaxCommittedTime(tabletTime.getTime());
+
+ if (count[0] == 0) {
+ log.debug("No replayed mutations applied, removing unused entries for " + extent);
+ MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
+
+ // No replication update to be made because the fact that this tablet didn't use any mutations
+ // from the WAL implies nothing about use of this WAL by other tablets. Do nothing.
+
+ logEntries.clear();
+ } else if (ReplicationConfigurationUtil.isEnabled(extent, tabletServer.getTableConfiguration(extent))) {
+ // The logs are about to be re-used by this tablet, we need to record that they have data for this extent,
+ // but that they may get more data. logEntries is not cleared which will cause the elements
+ // in logEntries to be added to the currentLogs for this Tablet below.
+ //
+ // This update serves the same purpose as an update during a MinC. We know that the WAL was defined
+ // (written when the WAL was opened) but this lets us know there are mutations written to this WAL
+ // that could potentially be replicated. Because the Tablet is using this WAL, we can be sure that
+ // the WAL isn't closed (WRT replication Status) and thus we're safe to update its progress.
+ Status status = StatusUtil.openWithUnknownLength();
+ for (LogEntry logEntry : logEntries) {
+ log.debug("Writing updated status to metadata table for " + logEntry.logSet + " " + ProtobufUtil.toString(status));
+ ReplicationTableUtil.updateFiles(SystemCredentials.get(), extent, logEntry.logSet, status);
+ }
+ }
+
+ } catch (Throwable t) {
+ if (tableConfiguration.getBoolean(Property.TABLE_FAILURES_IGNORE)) {
+ log.warn("Error recovering from log files: ", t);
+ } else {
+ throw new RuntimeException(t);
+ }
+ }
+ // make some closed references that represent the recovered logs
+ currentLogs = new HashSet<DfsLogger>();
+ for (LogEntry logEntry : logEntries) {
+ for (String log : logEntry.logSet) {
+ currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), log, logEntry.getColumnQualifier().toString()));
+ }
+ }
+
+ log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + getTabletMemory().getNumEntries()
+ + " entries created)");
+ }
+
+ String contextName = tableConfiguration.get(Property.TABLE_CLASSPATH);
+ if (contextName != null && !contextName.equals("")) {
+ // initialize context classloader, instead of possibly waiting for it to initialize for a scan
+ // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292
+ AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
+ }
+
+ // do this last after tablet is completely setup because it
+ // could cause major compaction to start
+ datafileManager = new DatafileManager(this, datafiles);
+
+ computeNumEntries();
+
+ getDatafileManager().removeFilesAfterScan(scanFiles);
+
+ // look for hints of a failure on the previous tablet server
+ if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
+ // look for any temp files hanging around
+ removeOldTemporaryFiles();
+ }
+
+ log.log(TLevel.TABLET_HIST, extent + " opened");
+ }
+
+ private void removeOldTemporaryFiles() {
+ // remove any temporary files created by a previous tablet server
+ try {
+ for (FileStatus tmp : getTabletServer().getFileSystem().globStatus(new Path(location, "*_tmp"))) {
+ try {
+ log.debug("Removing old temp file " + tmp.getPath());
+ getTabletServer().getFileSystem().delete(tmp.getPath());
+ } catch (IOException ex) {
+ log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
+ }
+ }
+ } catch (IOException ex) {
+ log.error("Error scanning for old temp files in " + location);
+ }
+ }
+
+ private void setupDefaultSecurityLabels(KeyExtent extent) {
+ if (extent.isMeta()) {
+ defaultSecurityLabel = new byte[0];
+ } else {
+ try {
+ ColumnVisibility cv = new ColumnVisibility(tableConfiguration.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
+ this.defaultSecurityLabel = cv.getExpression();
+ } catch (Exception e) {
+ log.error(e, e);
+ this.defaultSecurityLabel = new byte[0];
+ }
+ }
+ }
+
+ private LookupResult lookup(SortedKeyValueIterator<Key,Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, List<KVEntry> results, long maxResultsSize)
+ throws IOException {
+
+ LookupResult lookupResult = new LookupResult();
+
+ boolean exceededMemoryUsage = false;
+ boolean tabletClosed = false;
+
+ Set<ByteSequence> cfset = null;
+ if (columnSet.size() > 0)
+ cfset = LocalityGroupUtil.families(columnSet);
+
+ for (Range range : ranges) {
+
+ if (exceededMemoryUsage || tabletClosed) {
+ lookupResult.unfinishedRanges.add(range);
+ continue;
+ }
+
+ int entriesAdded = 0;
+
+ try {
+ if (cfset != null)
+ mmfi.seek(range, cfset, true);
+ else
+ mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
+
+ while (mmfi.hasTop()) {
+ Key key = mmfi.getTopKey();
+
+ KVEntry kve = new KVEntry(key, mmfi.getTopValue());
+ results.add(kve);
+ entriesAdded++;
+ lookupResult.bytesAdded += kve.estimateMemoryUsed();
+ lookupResult.dataSize += kve.numBytes();
+
+ exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
+
+ if (exceededMemoryUsage) {
+ addUnfinishedRange(lookupResult, range, key, false);
+ break;
+ }
+
+ mmfi.next();
+ }
+
+ } catch (TooManyFilesException tmfe) {
+ // treat this as a closed tablet, and let the client retry
+ log.warn("Tablet " + getExtent() + " has too many files, batch lookup can not run");
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } catch (IOException ioe) {
+ if (shutdownInProgress()) {
+ // assume HDFS shutdown hook caused this exception
+ log.debug("IOException while shutdown in progress ", ioe);
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw ioe;
+ }
+ } catch (IterationInterruptedException iie) {
+ if (isClosed()) {
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw iie;
+ }
+ } catch (TabletClosedException tce) {
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ }
+
+ }
+
+ return lookupResult;
+ }
+
+ private void handleTabletClosedDuringScan(List<KVEntry> results, LookupResult lookupResult, boolean exceededMemoryUsage, Range range, int entriesAdded) {
+ if (exceededMemoryUsage)
+ throw new IllegalStateException("tablet should not exceed memory usage or close, not both");
+
+ if (entriesAdded > 0)
+ addUnfinishedRange(lookupResult, range, results.get(results.size() - 1).getKey(), false);
+ else
+ lookupResult.unfinishedRanges.add(range);
+
+ lookupResult.closed = true;
+ }
+
+ private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) {
+ if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) {
+ Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive());
+ lookupResult.unfinishedRanges.add(nlur);
+ }
+ }
+
+ public LookupResult lookup(List<Range> ranges, HashSet<Column> columns, Authorizations authorizations, List<KVEntry> results, long maxResultSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag) throws IOException {
+
+ if (ranges.size() == 0) {
+ return new LookupResult();
+ }
+
+ ranges = Range.mergeOverlapping(ranges);
+ Collections.sort(ranges);
+
+ Range tabletRange = extent.toDataRange();
+ for (Range range : ranges) {
+ // do a test to see if this range falls within the tablet, if it does not
+ // then clip will throw an exception
+ tabletRange.clip(range);
+ }
+
+ ScanDataSource dataSource = new ScanDataSource(this, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
+
+ LookupResult result = null;
+
+ try {
+ SortedKeyValueIterator<Key,Value> iter = new SourceSwitchingIterator(dataSource);
+ result = lookup(iter, ranges, columns, results, maxResultSize);
+ return result;
+ } catch (IOException ioe) {
+ dataSource.close(true);
+ throw ioe;
+ } finally {
+ // code in finally block because always want
+ // to return mapfiles, even when exception is thrown
+ dataSource.close(false);
+
+ synchronized (this) {
+ queryCount += results.size();
+ if (result != null)
+ queryBytes += result.dataSize;
+ }
+ }
+ }
+
+ Batch nextBatch(SortedKeyValueIterator<Key,Value> iter, Range range, int num, Set<Column> columns) throws IOException {
+
+ // log.info("In nextBatch..");
+
+ List<KVEntry> results = new ArrayList<KVEntry>();
+ Key key = null;
+
+ Value value;
+ long resultSize = 0L;
+ long resultBytes = 0L;
+
+ long maxResultsSize = tableConfiguration.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ if (columns.size() == 0) {
+ iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
+ } else {
+ iter.seek(range, LocalityGroupUtil.families(columns), true);
+ }
+
+ Key continueKey = null;
+ boolean skipContinueKey = false;
+
+ boolean endOfTabletReached = false;
+ while (iter.hasTop()) {
+
+ value = iter.getTopValue();
+ key = iter.getTopKey();
+
+ KVEntry kvEntry = new KVEntry(key, value); // copies key and value
+ results.add(kvEntry);
+ resultSize += kvEntry.estimateMemoryUsed();
+ resultBytes += kvEntry.numBytes();
+
+ if (resultSize >= maxResultsSize || results.size() >= num) {
+ continueKey = new Key(key);
+ skipContinueKey = true;
+ break;
+ }
+
+ iter.next();
+ }
+
+ if (iter.hasTop() == false) {
+ endOfTabletReached = true;
+ }
+
+ if (endOfTabletReached) {
+ continueKey = null;
+ }
+
+ if (endOfTabletReached && results.size() == 0)
+ results = null;
+
+ return new Batch(skipContinueKey, results, continueKey, resultBytes);
+ }
+
+ /**
+ * Determine if a JVM shutdown is in progress.
+ *
+ */
+ boolean shutdownInProgress() {
+ try {
+ Runtime.getRuntime().removeShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {}
+ }));
+ } catch (IllegalStateException ise) {
+ return true;
+ }
+
+ return false;
+ }
+
+ public Scanner createScanner(Range range, int num, Set<Column> columns, Authorizations authorizations, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, boolean isolated, AtomicBoolean interruptFlag) {
+ // do a test to see if this range falls within the tablet, if it does not
+ // then clip will throw an exception
+ extent.toDataRange().clip(range);
+
+ ScanOptions opts = new ScanOptions(num, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag, isolated);
+ return new Scanner(this, range, opts);
+ }
+
+ DataFileValue minorCompact(VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile, boolean hasQueueTime,
+ long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
+ boolean failed = false;
+ long start = System.currentTimeMillis();
+ timer.incrementStatusMinor();
+
+ long count = 0;
+
+ try {
+ Span span = Trace.start("write");
+ CompactionStats stats;
+ try {
+ count = memTable.getNumEntries();
+
+ DataFileValue dfv = null;
+ if (mergeFile != null)
+ dfv = getDatafileManager().getDatafileSizes().get(mergeFile);
+
+ MinorCompactor compactor = new MinorCompactor(this, memTable, mergeFile, dfv, tmpDatafile, mincReason, tableConfiguration);
+ stats = compactor.call();
+ } finally {
+ span.stop();
+ }
+ span = Trace.start("bringOnline");
+ try {
+ getDatafileManager().bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
+ commitSession, flushId);
+ } finally {
+ span.stop();
+ }
+ return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
+ } catch (Exception e) {
+ failed = true;
+ throw new RuntimeException(e);
+ } catch (Error e) {
+ // Weird errors like "OutOfMemoryError" when trying to create the thread for the compaction
+ failed = true;
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ getTabletMemory().finalizeMinC();
+ } catch (Throwable t) {
+ log.error("Failed to free tablet memory", t);
+ }
+
+ if (!failed) {
+ lastMinorCompactionFinishTime = System.currentTimeMillis();
+ }
+ TabletServerMinCMetrics minCMetrics = getTabletServer().getMinCMetrics();
+ if (minCMetrics.isEnabled())
+ minCMetrics.add(TabletServerMinCMetrics.minc, (lastMinorCompactionFinishTime - start));
+ if (hasQueueTime) {
+ timer.updateTime(Operation.MINOR, queued, start, count, failed);
+ if (minCMetrics.isEnabled())
+ minCMetrics.add(TabletServerMinCMetrics.queue, (start - queued));
+ } else
+ timer.updateTime(Operation.MINOR, start, count, failed);
+ }
+ }
+
+ private synchronized MinorCompactionTask prepareForMinC(long flushId, MinorCompactionReason mincReason) {
+ CommitSession oldCommitSession = getTabletMemory().prepareForMinC();
+ otherLogs = currentLogs;
+ currentLogs = new HashSet<DfsLogger>();
+
+ FileRef mergeFile = null;
+ if (mincReason != MinorCompactionReason.RECOVERY) {
+ mergeFile = getDatafileManager().reserveMergingMinorCompactionFile();
+ }
+
+ return new MinorCompactionTask(this, mergeFile, oldCommitSession, flushId, mincReason);
+
+ }
+
+ public void flush(long tableFlushID) {
+ boolean updateMetadata = false;
+ boolean initiateMinor = false;
+
+ try {
+
+ synchronized (this) {
+
+ // only want one thing at a time to update flush ID to ensure that metadata table and tablet in memory state are consistent
+ if (updatingFlushID)
+ return;
+
+ if (lastFlushID >= tableFlushID)
+ return;
+
+ if (isClosing() || isClosed() || getTabletMemory().memoryReservedForMinC())
+ return;
+
+ if (getTabletMemory().getMemTable().getNumEntries() == 0) {
+ lastFlushID = tableFlushID;
+ updatingFlushID = true;
+ updateMetadata = true;
+ } else
+ initiateMinor = true;
+ }
+
+ if (updateMetadata) {
+ Credentials creds = SystemCredentials.get();
+ // if multiple threads were allowed to update this outside of a sync block, then it would be
+ // a race condition
+ MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, getTabletServer().getLock());
+ } else if (initiateMinor)
+ initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
+
+ } finally {
+ if (updateMetadata) {
+ synchronized (this) {
+ updatingFlushID = false;
+ this.notifyAll();
+ }
+ }
+ }
+
+ }
+
+ public boolean initiateMinorCompaction(MinorCompactionReason mincReason) {
+ if (isClosed()) {
+ // don't bother trying to get flush id if closed... could be closed after this check but that is ok... just trying to cut down on uneeded log messages....
+ return false;
+ }
+
+ // get the flush id before the new memmap is made available for write
+ long flushId;
+ try {
+ flushId = getFlushID();
+ } catch (NoNodeException e) {
+ log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
+ return false;
+ }
+ return initiateMinorCompaction(flushId, mincReason);
+ }
+
+ public boolean minorCompactNow(MinorCompactionReason mincReason) {
+ long flushId;
+ try {
+ flushId = getFlushID();
+ } catch (NoNodeException e) {
+ log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
+ return false;
+ }
+ MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
+ if (mct == null)
+ return false;
+ mct.run();
+ return true;
+ }
+
+ boolean initiateMinorCompaction(long flushId, MinorCompactionReason mincReason) {
+ MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
+ if (mct == null)
+ return false;
+ getTabletResources().executeMinorCompaction(mct);
+ return true;
+ }
+
+ private MinorCompactionTask createMinorCompactionTask(long flushId, MinorCompactionReason mincReason) {
+ MinorCompactionTask mct;
+ long t1, t2;
+
+ StringBuilder logMessage = null;
+
+ try {
+ synchronized (this) {
+ t1 = System.currentTimeMillis();
+
+ if (isClosing() || isClosed() || majorCompactionState == CompactionState.WAITING_TO_START || getTabletMemory().memoryReservedForMinC()
+ || getTabletMemory().getMemTable().getNumEntries() == 0 || updatingFlushID) {
+
+ logMessage = new StringBuilder();
+
+ logMessage.append(extent.toString());
+ logMessage.append(" closeState " + closeState);
+ logMessage.append(" majorCompactionState " + majorCompactionState);
+ if (getTabletMemory() != null)
+ logMessage.append(" tabletMemory.memoryReservedForMinC() " + getTabletMemory().memoryReservedForMinC());
+ if (getTabletMemory() != null && getTabletMemory().getMemTable() != null)
+ logMessage.append(" tabletMemory.getMemTable().getNumEntries() " + getTabletMemory().getMemTable().getNumEntries());
+ logMessage.append(" updatingFlushID " + updatingFlushID);
+
+ return null;
+ }
+
+ mct = prepareForMinC(flushId, mincReason);
+ t2 = System.currentTimeMillis();
+ }
+ } finally {
+ // log outside of sync block
+ if (logMessage != null && log.isDebugEnabled())
+ log.debug(logMessage);
+ }
+
+ log.debug(String.format("MinC initiate lock %.2f secs", (t2 - t1) / 1000.0));
+ return mct;
+ }
+
+ public long getFlushID() throws NoNodeException {
+ try {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_FLUSH_ID;
- return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), StandardCharsets.UTF_8));
++ return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (NumberFormatException nfe) {
+ throw new RuntimeException(nfe);
+ } catch (KeeperException ke) {
+ if (ke instanceof NoNodeException) {
+ throw (NoNodeException) ke;
+ } else {
+ throw new RuntimeException(ke);
+ }
+ }
+ }
+
+ long getCompactionCancelID() {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ try {
- return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), StandardCharsets.UTF_8));
++ return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
+ } catch (KeeperException e) {
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Pair<Long,List<IteratorSetting>> getCompactionID() throws NoNodeException {
+ try {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_COMPACT_ID;
+
- String[] tokens = new String(ZooReaderWriter.getInstance().getData(zTablePath, null), StandardCharsets.UTF_8).split(",");
++ String[] tokens = new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8).split(",");
+ long compactID = Long.parseLong(tokens[0]);
+
+ CompactionIterators iters = new CompactionIterators();
+
+ if (tokens.length > 1) {
+ Hex hex = new Hex();
- ByteArrayInputStream bais = new ByteArrayInputStream(hex.decode(tokens[1].split("=")[1].getBytes(StandardCharsets.UTF_8)));
++ ByteArrayInputStream bais = new ByteArrayInputStream(hex.decode(tokens[1].split("=")[1].getBytes(UTF_8)));
+ DataInputStream dis = new DataInputStream(bais);
+
+ try {
+ iters.readFields(dis);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ KeyExtent ke = new KeyExtent(extent.getTableId(), iters.getEndRow(), iters.getStartRow());
+
+ if (!ke.overlaps(extent)) {
+ // only use iterators if compaction range overlaps
+ iters = new CompactionIterators();
+ }
+ }
+
+ return new Pair<Long,List<IteratorSetting>>(compactID, iters.getIterators());
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (NumberFormatException nfe) {
+ throw new RuntimeException(nfe);
+ } catch (KeeperException ke) {
+ if (ke instanceof NoNodeException) {
+ throw (NoNodeException) ke;
+ } else {
+ throw new RuntimeException(ke);
+ }
+ } catch (DecoderException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public synchronized void waitForMinC() {
+ getTabletMemory().waitForMinC();
+ }
+
+ private synchronized CommitSession finishPreparingMutations(long time) {
+ if (writesInProgress < 0) {
+ throw new IllegalStateException("waitingForLogs < 0 " + writesInProgress);
+ }
+
+ if (isClosed() || getTabletMemory() == null) {
+ return null;
+ }
+
+ writesInProgress++;
+ CommitSession commitSession = getTabletMemory().getCommitSession();
+ commitSession.incrementCommitsInProgress();
+ commitSession.updateMaxCommittedTime(time);
+ return commitSession;
+ }
+
+ public void checkConstraints() {
+ ConstraintChecker cc = constraintChecker.get();
+
+ if (cc.classLoaderChanged()) {
+ ConstraintChecker ncc = new ConstraintChecker(tableConfiguration);
+ constraintChecker.compareAndSet(cc, ncc);
+ }
+ }
+
+ public CommitSession prepareMutationsForCommit(TservConstraintEnv cenv, List<Mutation> mutations) throws TConstraintViolationException {
+
+ ConstraintChecker cc = constraintChecker.get();
+
+ List<Mutation> violators = null;
+ Violations violations = new Violations();
+ cenv.setExtent(extent);
+ for (Mutation mutation : mutations) {
+ Violations more = cc.check(cenv, mutation);
+ if (more != null) {
+ violations.add(more);
+ if (violators == null)
+ violators = new ArrayList<Mutation>();
+ violators.add(mutation);
+ }
+ }
+
+ long time = tabletTime.setUpdateTimes(mutations);
+
+ if (!violations.isEmpty()) {
+
+ HashSet<Mutation> violatorsSet = new HashSet<Mutation>(violators);
+ ArrayList<Mutation> nonViolators = new ArrayList<Mutation>();
+
+ for (Mutation mutation : mutations) {
+ if (!violatorsSet.contains(mutation)) {
+ nonViolators.add(mutation);
+ }
+ }
+
+ CommitSession commitSession = null;
+
+ if (nonViolators.size() > 0) {
+ // if everything is a violation, then it is expected that
+ // code calling this will not log or commit
+ commitSession = finishPreparingMutations(time);
+ if (commitSession == null)
+ return null;
+ }
+
+ throw new TConstraintViolationException(violations, violators, nonViolators, commitSession);
+ }
+
+ return finishPreparingMutations(time);
+ }
+
+ @Override
+ public synchronized void abortCommit(CommitSession commitSession, List<Mutation> value) {
+ if (writesInProgress <= 0) {
+ throw new IllegalStateException("waitingForLogs <= 0 " + writesInProgress);
+ }
+
+ if (isCloseComplete() || getTabletMemory() == null) {
+ throw new IllegalStateException("aborting commit when tablet is closed");
+ }
+
+ commitSession.decrementCommitsInProgress();
+ writesInProgress--;
+ if (writesInProgress == 0)
+ this.notifyAll();
+ }
+
+ @Override
+ public void commit(CommitSession commitSession, List<Mutation> mutations) {
+
+ int totalCount = 0;
+ long totalBytes = 0;
+
+ // write the mutation to the in memory table
+ for (Mutation mutation : mutations) {
+ totalCount += mutation.size();
+ totalBytes += mutation.numBytes();
+ }
+
+ getTabletMemory().mutate(commitSession, mutations);
+
+ synchronized (this) {
+ if (writesInProgress < 1) {
+ throw new IllegalStateException("commiting mutations after logging, but not waiting for any log messages");
+ }
+
+ if (isCloseComplete()) {
+ throw new IllegalStateException("tablet closed with outstanding messages to the logger");
+ }
+
+ getTabletMemory().updateMemoryUsageStats();
+
+ // decrement here in case an exception is thrown below
+ writesInProgress--;
+ if (writesInProgress == 0)
+ this.notifyAll();
+
+ commitSession.decrementCommitsInProgress();
+
+ numEntries += totalCount;
+ numEntriesInMemory += totalCount;
+ ingestCount += totalCount;
+ ingestBytes += totalBytes;
+ }
+ }
+
+ /**
+ * Closes the mapfiles associated with a Tablet. If saveState is true, a minor compaction is performed.
+ */
+ public void close(boolean saveState) throws IOException {
+ initiateClose(saveState, false, false);
+ completeClose(saveState, true);
+ }
+
+ void initiateClose(boolean saveState, boolean queueMinC, boolean disableWrites) {
+
+ if (!saveState && queueMinC) {
+ throw new IllegalArgumentException("Not saving state on close and requesting minor compactions queue does not make sense");
+ }
+
+ log.debug("initiateClose(saveState=" + saveState + " queueMinC=" + queueMinC + " disableWrites=" + disableWrites + ") " + getExtent());
+
+ MinorCompactionTask mct = null;
+
+ synchronized (this) {
+ if (isClosed() || isClosing()) {
+ String msg = "Tablet " + getExtent() + " already " + closeState;
+ throw new IllegalStateException(msg);
+ }
+
+ // enter the closing state, no splits, minor, or major compactions can start
+ // should cause running major compactions to stop
+ closeState = CloseState.CLOSING;
+ this.notifyAll();
+
+ // determines if inserts and queries can still continue while minor compacting
+ if (disableWrites) {
+ closeState = CloseState.CLOSED;
+ }
+
+ // wait for major compactions to finish, setting closing to
+ // true should cause any running major compactions to abort
+ while (isMajorCompactionRunning()) {
+ try {
+ this.wait(50);
+ } catch (InterruptedException e) {
+ log.error(e.toString());
+ }
+ }
+
+ while (updatingFlushID) {
+ try {
+ this.wait(50);
+ } catch (InterruptedException e) {
+ log.error(e.toString());
+ }
+ }
+
+ if (!saveState || getTabletMemory().getMemTable().getNumEntries() == 0) {
+ return;
+ }
+
+ getTabletMemory().waitForMinC();
+
+ try {
+ mct = prepareForMinC(getFlushID(), MinorCompactionReason.CLOSE);
+ } catch (NoNodeException e) {
+ throw new RuntimeException(e);
+ }
+
+ if (queueMinC) {
+ getTabletResources().executeMinorCompaction(mct);
+ return;
+ }
+
+ }
+
+ // do minor compaction outside of synch block so that tablet can be read and written to while
+ // compaction runs
+ mct.run();
+ }
+
+ private boolean closeCompleting = false;
+
+ synchronized void completeClose(boolean saveState, boolean completeClose) throws IOException {
+
+ if (!isClosing() || isCloseComplete() || closeCompleting) {
+ throw new IllegalStateException("closeState = " + closeState);
+ }
+
+ log.debug("completeClose(saveState=" + saveState + " completeClose=" + completeClose + ") " + getExtent());
+
+ // ensure this method is only called once, also guards against multiple
+ // threads entering the method at the same time
+ closeCompleting = true;
+ closeState = CloseState.CLOSED;
+
+ // modify dataSourceDeletions so scans will try to switch data sources and fail because the tablet is closed
+ dataSourceDeletions.incrementAndGet();
+
+ for (ScanDataSource activeScan : activeScans) {
+ activeScan.interrupt();
+ }
+
+ // wait for reads and writes to complete
+ while (writesInProgress > 0 || activeScans.size() > 0) {
+ try {
+ this.wait(50);
+ } catch (InterruptedException e) {
+ log.error(e.toString());
+ }
+ }
+
+ getTabletMemory().waitForMinC();
+
+ if (saveState && getTabletMemory().getMemTable().getNumEntries() > 0) {
+ try {
+ prepareForMinC(getFlushID(), MinorCompactionReason.CLOSE).run();
+ } catch (NoNodeException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ if (saveState) {
+ // at this point all tablet data is flushed, so do a consistency check
+ RuntimeException err = null;
+ for (int i = 0; i < 5; i++) {
+ try {
+ closeConsistencyCheck();
+ err = null;
+ } catch (RuntimeException t) {
+ err = t;
+ log.error("Consistency check fails, retrying " + t);
+ UtilWaitThread.sleep(500);
+ }
+ }
+ if (err != null) {
+ ProblemReports.getInstance().report(new ProblemReport(extent.getTableId().toString(), ProblemType.TABLET_LOAD, this.extent.toString(), err));
+ log.error("Tablet closed consistency check has failed for " + this.extent + " giving up and closing");
+ }
+ }
+
+ try {
+ getTabletMemory().getMemTable().delete(0);
+ } catch (Throwable t) {
+ log.error("Failed to delete mem table : " + t.getMessage(), t);
+ }
+
+ getTabletMemory().close();
+
+ // close map files
+ getTabletResources().close();
+
+ log.log(TLevel.TABLET_HIST, extent + " closed");
+
+ tableConfiguration.getNamespaceConfiguration().removeObserver(configObserver);
+ tableConfiguration.removeObserver(configObserver);
+
+ if (completeClose)
+ closeState = CloseState.COMPLETE;
+ }
+
+ private void closeConsistencyCheck() {
+
+ if (getTabletMemory().getMemTable().getNumEntries() != 0) {
+ String msg = "Closed tablet " + extent + " has " + getTabletMemory().getMemTable().getNumEntries() + " entries in memory";
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ if (getTabletMemory().memoryReservedForMinC()) {
+ String msg = "Closed tablet " + extent + " has minor compacting memory";
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ try {
+ Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> fileLog = MetadataTableUtil.getFileAndLogEntries(SystemCredentials.get(), extent);
+
+ if (fileLog.getFirst().size() != 0) {
+ String msg = "Closed tablet " + extent + " has walog entries in " + MetadataTable.NAME + " " + fileLog.getFirst();
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ if (extent.isRootTablet()) {
+ if (!fileLog.getSecond().keySet().equals(getDatafileManager().getDatafileSizes().keySet())) {
+ String msg = "Data file in " + RootTable.NAME + " differ from in memory data " + extent + " " + fileLog.getSecond().keySet() + " "
+ + getDatafileManager().getDatafileSizes().keySet();
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+ } else {
+ if (!fileLog.getSecond().equals(getDatafileManager().getDatafileSizes())) {
+ String msg = "Data file in " + MetadataTable.NAME + " differ from in memory data " + extent + " " + fileLog.getSecond() + " "
+ + getDatafileManager().getDatafileSizes();
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+ }
+
+ } catch (Exception e) {
+ String msg = "Failed to do close consistency check for tablet " + extent;
+ log.error(msg, e);
+ throw new RuntimeException(msg, e);
+
+ }
+
+ if (otherLogs.size() != 0 || currentLogs.size() != 0) {
+ String msg = "Closed tablet " + extent + " has walog entries in memory currentLogs = " + currentLogs + " otherLogs = " + otherLogs;
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ // TODO check lastFlushID and lostCompactID - ACCUMULO-1290
+ }
+
+ /**
+ * Returns a Path object representing the tablet's location on the DFS.
+ *
+ * @return location
+ */
+ public Path getLocation() {
+ return location;
+ }
+
+ public synchronized boolean initiateMajorCompaction(MajorCompactionReason reason) {
+
+ if (isClosing() || isClosed() || !needsMajorCompaction(reason) || isMajorCompactionRunning() || majorCompactionQueued.contains(reason)) {
+ return false;
+ }
+
+ majorCompactionQueued.add(reason);
+
+ getTabletResources().executeMajorCompaction(getExtent(), new CompactionRunner(this, reason));
+
+ return false;
+ }
+
+ /**
+ * Returns true if a major compaction should be performed on the tablet.
+ *
+ */
+ public boolean needsMajorCompaction(MajorCompactionReason reason) {
+ if (isMajorCompactionRunning())
+ return false;
+ if (reason == MajorCompactionReason.CHOP || reason == MajorCompactionReason.USER)
+ return true;
+ return getTabletResources().needsMajorCompaction(getDatafileManager().getDatafileSizes(), reason);
+ }
+
+ /**
+ * Returns an int representing the total block size of the files served by this tablet.
+ *
+ * @return size
+ */
+ // this is the size of just the files
+ public long estimateTabletSize() {
+ long size = 0L;
+
+ for (DataFileValue sz : getDatafileManager().getDatafileSizes().values())
+ size += sz.getSize();
+
+ return size;
+ }
+
+ private boolean sawBigRow = false;
+ private long timeOfLastMinCWhenBigFreakinRowWasSeen = 0;
+ private long timeOfLastImportWhenBigFreakinRowWasSeen = 0;
+ private long splitCreationTime;
+
+ private SplitRowSpec findSplitRow(Collection<FileRef> files) {
+
+ // never split the root tablet
+ // check if we already decided that we can never split
+ // check to see if we're big enough to split
+
+ long splitThreshold = tableConfiguration.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD);
+ long maxEndRow = tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE);
+
+ if (extent.isRootTablet() || estimateTabletSize() <= splitThreshold) {
+ return null;
+ }
+
+ // have seen a big row before, do not bother checking unless a minor compaction or map file import has occurred.
+ if (sawBigRow) {
+ if (timeOfLastMinCWhenBigFreakinRowWasSeen != lastMinorCompactionFinishTime || timeOfLastImportWhenBigFreakinRowWasSeen != lastMapFileImportTime) {
+ // a minor compaction or map file import has occurred... check again
+ sawBigRow = false;
+ } else {
+ // nothing changed, do not split
+ return null;
+ }
+ }
+
+ SortedMap<Double,Key> keys = null;
+
+ try {
+ // we should make .25 below configurable
+ keys = FileUtil.findMidPoint(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), extent.getPrevEndRow(), extent.getEndRow(),
+ FileUtil.toPathStrings(files), .25);
+ } catch (IOException e) {
+ log.error("Failed to find midpoint " + e.getMessage());
+ return null;
+ }
+
+ // check to see if one row takes up most of the tablet, in which case we can not split
+ try {
+
+ Text lastRow;
+ if (extent.getEndRow() == null) {
+ Key lastKey = (Key) FileUtil.findLastKey(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), files);
+ lastRow = lastKey.getRow();
+ } else {
+ lastRow = extent.getEndRow();
+ }
+
+ // We expect to get a midPoint for this set of files. If we don't get one, we have a problem.
+ final Key mid = keys.get(.5);
+ if (null == mid) {
+ throw new IllegalStateException("Could not determine midpoint for files");
+ }
+
+ // check to see that the midPoint is not equal to the end key
+ if (mid.compareRow(lastRow) == 0) {
+ if (keys.firstKey() < .5) {
+ Key candidate = keys.get(keys.firstKey());
+ if (candidate.getLength() > maxEndRow) {
+ log.warn("Cannot split tablet " + extent + ", selected split point too long. Length : " + candidate.getLength());
+
+ sawBigRow = true;
+ timeOfLastMinCWhenBigFreakinRowWasSeen = lastMinorCompactionFinishTime;
+ timeOfLastImportWhenBigFreakinRowWasSeen = lastMapFileImportTime;
+
+ return null;
+ }
+ if (candidate.compareRow(lastRow) != 0) {
+ // we should use this ratio in split size estimations
+ if (log.isTraceEnabled())
+ log.trace(String.format("Splitting at %6.2f instead of .5, row at .5 is same as end row%n", keys.firstKey()));
+ return new SplitRowSpec(keys.firstKey(), candidate.getRow());
+ }
+
+ }
+
+ log.warn("Cannot split tablet " + extent + " it contains a big row : " + lastRow);
+
+ sawBigRow = true;
+ timeOfLastMinCWhenBigFreakinRowWasSeen = lastMinorCompactionFinishTime;
+ timeOfLastImportWhenBigFreakinRowWasSeen = lastMapFileImportTime;
+
+ return null;
+ }
+
+ Text text = mid.getRow();
+ SortedMap<Double,Key> firstHalf = keys.headMap(.5);
+ if (firstHalf.size() > 0) {
+ Text beforeMid = firstHalf.get(firstHalf.lastKey()).getRow();
+ Text shorter = new Text();
+ int trunc = longestCommonLength(text, beforeMid);
+ shorter.set(text.getBytes(), 0, Math.min(text.getLength(), trunc + 1));
+ text = shorter;
+ }
+
+ if (text.getLength() > maxEndRow) {
+ log.warn("Cannot split tablet " + extent + ", selected split point too long. Length : " + text.getLength());
+
+ sawBigRow = true;
+ timeOfLastMinCWhenBigFreakinRowWasSeen = lastMinorCompactionFinishTime;
+ timeOfLastImportWhenBigFreakinRowWasSeen = lastMapFileImportTime;
+
+ return null;
+ }
+
+ return new SplitRowSpec(.5, text);
+ } catch (IOException e) {
+ // don't split now, but check again later
+ log.error("Failed to find lastkey " + e.getMessage());
+ return null;
+ }
+
+ }
+
+ private static int longestCommonLength(Text text, Text beforeMid) {
+ int common = 0;
+ while (common < text.getLength() && common < beforeMid.getLength() && text.getBytes()[common] == beforeMid.getBytes()[common]) {
+ common++;
+ }
+ return common;
+ }
+
+ private Map<FileRef,Pair<Key,Key>> getFirstAndLastKeys(SortedMap<FileRef,DataFileValue> allFiles) throws IOException {
+ Map<FileRef,Pair<Key,Key>> result = new HashMap<FileRef,Pair<Key,Key>>();
+ FileOperations fileFactory = FileOperations.getInstance();
+ VolumeManager fs = getTabletServer().getFileSystem();
+ for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) {
+ FileRef file = entry.getKey();
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
+ FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration());
+ try {
+ Key first = openReader.getFirstKey();
+ Key last = openReader.getLastKey();
+ result.put(file, new Pair<Key,Key>(first, last));
+ } finally {
+ openReader.close();
+ }
+ }
+ return result;
+ }
+
+ List<FileRef> findChopFiles(KeyExtent extent, Map<FileRef,Pair<Key,Key>> firstAndLastKeys, Collection<FileRef> allFiles) throws IOException {
+ List<FileRef> result = new ArrayList<FileRef>();
+ if (firstAndLastKeys == null) {
+ result.addAll(allFiles);
+ return result;
+ }
+
+ for (FileRef file : allFiles) {
+ Pair<Key,Key> pair = firstAndLastKeys.get(file);
+ if (pair == null) {
+ // file was created or imported after we obtained the first and last keys... there
+ // are a few options here... throw an exception which will cause the compaction to
+ // retry and also cause ugly error message that the admin has to ignore... could
+ // go get the first and last key, but this code is called while the tablet lock
+ // is held... or just compact the file....
+ result.add(file);
+ } else {
+ Key first = pair.getFirst();
+ Key last = pair.getSecond();
+ // If first and last are null, it's an empty file. Add it to the compact set so it goes away.
+ if ((first == null && last == null) || !extent.contains(first.getRow()) || !extent.contains(last.getRow())) {
+ result.add(file);
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns true if this tablet needs to be split
+ *
+ */
+ public synchronized boolean needsSplit() {
+ if (isClosing() || isClosed())
+ return false;
+ return findSplitRow(getDatafileManager().getFiles()) != null;
+ }
+
+ // BEGIN PRIVATE METHODS RELATED TO MAJOR COMPACTION
+
+ private boolean isCompactionEnabled() {
+ return !isClosing() && !getTabletServer().isMajorCompactionDisabled();
+ }
+
+ private CompactionStats _majorCompact(MajorCompactionReason reason) throws IOException, CompactionCanceledException {
+
+ long t1, t2, t3;
+
+ // acquire file info outside of tablet lock
+ CompactionStrategy strategy = Property.createTableInstanceFromPropertyName(tableConfiguration, Property.TABLE_COMPACTION_STRATEGY,
+ CompactionStrategy.class, new DefaultCompactionStrategy());
+ strategy.init(Property.getCompactionStrategyOptions(tableConfiguration));
+
+ Map<FileRef,Pair<Key,Key>> firstAndLastKeys = null;
+ if (reason == MajorCompactionReason.CHOP) {
+ firstAndLastKeys = getFirstAndLastKeys(getDatafileManager().getDatafileSizes());
+ } else if (reason != MajorCompactionReason.USER) {
+ MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, getTabletServer().getFileSystem(), tableConfiguration);
+ request.setFiles(getDatafileManager().getDatafileSizes());
+ strategy.gatherInformation(request);
+ }
+
+ Map<FileRef,DataFileValue> filesToCompact;
+
+ int maxFilesToCompact = tableConfiguration.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
+
+ CompactionStats majCStats = new CompactionStats();
+ CompactionPlan plan = null;
+
+ boolean propogateDeletes = false;
+
+ synchronized (this) {
+ // plan all that work that needs to be done in the sync block... then do the actual work
+ // outside the sync block
+
+ t1 = System.currentTimeMillis();
+
+ majorCompactionState = CompactionState.WAITING_TO_START;
+
+ getTabletMemory().waitForMinC();
+
+ t2 = System.currentTimeMillis();
+
+ majorCompactionState = CompactionState.IN_PROGRESS;
+ notifyAll();
+
+ VolumeManager fs = getTabletServer().getFileSystem();
+ if (extent.isRootTablet()) {
+ // very important that we call this before doing major compaction,
+ // otherwise deleted compacted files could possible be brought back
+ // at some point if the file they were compacted to was legitimately
+ // removed by a major compaction
+ RootFiles.cleanupReplacement(fs, fs.listStatus(this.location), false);
+ }
+ SortedMap<FileRef,DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
+ List<FileRef> inputFiles = new ArrayList<FileRef>();
+ if (reason == MajorCompactionReason.CHOP) {
+ // enforce rules: files with keys outside our range need to be compacted
+ inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
+ } else if (reason == MajorCompactionReason.USER) {
+ inputFiles.addAll(allFiles.keySet());
+ } else {
+ MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, fs, tableConfiguration);
+ request.setFiles(allFiles);
+ plan = strategy.getCompactionPlan(request);
+ if (plan != null) {
+ plan.validate(allFiles.keySet());
+ inputFiles.addAll(plan.inputFiles);
+ }
+ }
+
+ if (inputFiles.isEmpty()) {
+ return majCStats;
+ }
+ // If no original files will exist at the end of the compaction, we do not have to propogate deletes
+ Set<FileRef> droppedFiles = new HashSet<FileRef>();
+ droppedFiles.addAll(inputFiles);
+ if (plan != null)
+ droppedFiles.addAll(plan.deleteFiles);
+ propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
+ log.debug("Major compaction plan: " + plan + " propogate deletes : " + propogateDeletes);
+ filesToCompact = new HashMap<FileRef,DataFileValue>(allFiles);
+ filesToCompact.keySet().retainAll(inputFiles);
+
+ t3 = System.currentTimeMillis();
+
+ getDatafileManager().reserveMajorCompactingFiles(filesToCompact.keySet());
+ }
+
+ try {
+
+ log.debug(String.format("MajC initiate lock %.2f secs, wait %.2f secs", (t3 - t2) / 1000.0, (t2 - t1) / 1000.0));
+
+ Pair<Long,List<IteratorSetting>> compactionId = null;
+ if (!propogateDeletes) {
+ // compacting everything, so update the compaction id in metadata
+ try {
+ compactionId = getCompactionID();
+ } catch (NoNodeException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ List<IteratorSetting> compactionIterators = new ArrayList<IteratorSetting>();
+ if (compactionId != null) {
+ if (reason == MajorCompactionReason.USER) {
+ if (getCompactionCancelID() >= compactionId.getFirst()) {
+ // compaction was canceled
+ return majCStats;
+ }
+
+ synchronized (this) {
+ if (lastCompactID >= compactionId.getFirst())
+ // already compacted
+ return majCStats;
+ }
+ }
+
+ compactionIterators = compactionId.getSecond();
+ }
+
+ // need to handle case where only one file is being major compacted
+ while (filesToCompact.size() > 0) {
+
+ int numToCompact = maxFilesToCompact;
+
+ if (filesToCompact.size() > maxFilesToCompact && filesToCompact.size() < 2 * maxFilesToCompact) {
+ // on the second to last compaction pass, compact the minimum amount of files possible
+ numToCompact = filesToCompact.size() - maxFilesToCompact + 1;
+ }
+
+ Set<FileRef> smallestFiles = removeSmallest(filesToCompact, numToCompact);
+
+ FileRef fileName = getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
+ FileRef compactTmpName = new FileRef(fileName.path().toString() + "_tmp");
+
+ AccumuloConfiguration tableConf = createTableConfiguration(tableConfiguration, plan);
+
+ Span span = Trace.start("compactFiles");
+ try {
+
+ CompactionEnv cenv = new CompactionEnv() {
+ @Override
+ public boolean isCompactionEnabled() {
+ return Tablet.this.isCompactionEnabled();
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return IteratorScope.majc;
+ }
+ };
+
+ HashMap<FileRef,DataFileValue> copy = new HashMap<FileRef,DataFileValue>(getDatafileManager().getDatafileSizes());
+ if (!copy.keySet().containsAll(smallestFiles))
+ throw new IllegalStateException("Cannot find data file values for " + smallestFiles);
+
+ copy.keySet().retainAll(smallestFiles);
+
+ log.debug("Starting MajC " + extent + " (" + reason + ") " + copy.keySet() + " --> " + compactTmpName + " " + compactionIterators);
+
+ // always propagate deletes, unless last batch
+ boolean lastBatch = filesToCompact.isEmpty();
+ Compactor compactor = new Compactor(this, copy, null, compactTmpName, lastBatch ? propogateDeletes : true, cenv, compactionIterators,
+ reason.ordinal(), tableConf);
+
+ CompactionStats mcs = compactor.call();
+
+ span.data("files", "" + smallestFiles.size());
+ span.data("read", "" + mcs.getEntriesRead());
+ span.data("written", "" + mcs.getEntriesWritten());
+ majCStats.add(mcs);
+
+ if (lastBatch && plan != null && plan.deleteFiles != null) {
+ smallestFiles.addAll(plan.deleteFiles);
+ }
+ getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
+ filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null,
+ new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
+
+ // when major compaction produces a file w/ zero entries, it will be deleted... do not want
+ // to add the deleted file
+ if (filesToCompact.size() > 0 && mcs.getEntriesWritten() > 0) {
+ filesToCompact.put(fileName, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
+ }
+ } finally {
+ span.stop();
+ }
+
+ }
+ return majCStats;
+ } finally {
+ synchronized (Tablet.this) {
+ getDatafileManager().clearMajorCompactingFile();
+ }
+ }
+ }
+
+ protected AccumuloConfiguration createTableConfiguration(TableConfiguration base, CompactionPlan plan) {
+ if (plan == null || plan.writeParameters == null)
+ return base;
+ WriteParameters p = plan.writeParameters;
+ ConfigurationCopy result = new ConfigurationCopy(base);
+ if (p.getHdfsBlockSize() > 0)
+ result.set(Property.TABLE_FILE_BLOCK_SIZE, "" + p.getHdfsBlockSize());
+ if (p.getBlockSize() > 0)
+ result.set(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE, "" + p.getBlockSize());
+ if (p.getIndexBlockSize() > 0)
+ result.set(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX, "" + p.getIndexBlockSize());
+ if (p.getCompressType() != null)
+ result.set(Property.TABLE_FILE_COMPRESSION_TYPE, p.getCompressType());
+ if (p.getReplication() != 0)
+ result.set(Property.TABLE_FILE_REPLICATION, "" + p.getReplication());
+ return result;
+ }
+
+ private Set<FileRef> removeSmallest(Map<FileRef,DataFileValue> filesToCompact, int maxFilesToCompact) {
+ // ensure this method works properly when multiple files have the same size
+
+ PriorityQueue<Pair<FileRef,Long>> fileHeap = new PriorityQueue<Pair<FileRef,Long>>(filesToCompact.size(), new Comparator<Pair<FileRef,Long>>() {
+ @Override
+ public int compare(Pair<FileRef,Long> o1, Pair<FileRef,Long> o2) {
+ if (o1.getSecond() == o2.getSecond())
+ return o1.getFirst().compareTo(o2.getFirst());
+ if (o1.getSecond() < o2.getSecond())
+ return -1;
+ return 1;
+ }
+ });
+
+ for (Iterator<Entry<FileRef,DataFileValue>> iterator = filesToCompact.entrySet().iterator(); iterator.hasNext();) {
+ Entry<FileRef,DataFileValue> entry = iterator.next();
+ fileHeap.add(new Pair<FileRef,Long>(entry.getKey(), entry.getValue().getSize()));
+ }
+
+ Set<FileRef> smallestFiles = new HashSet<FileRef>();
+ while (smallestFiles.size() < maxFilesToCompact && fileHeap.size() > 0) {
+ Pair<FileRef,Long> pair = fileHeap.remove();
+ filesToCompact.remove(pair.getFirst());
+ smallestFiles.add(pair.getFirst());
+ }
+
+ return smallestFiles;
+ }
+
+ // END PRIVATE METHODS RELATED TO MAJOR COMPACTION
+
+ /**
+ * Performs a major compaction on the tablet. If needsSplit() returns true, the tablet is split and a reference to the new tablet is returned.
+ */
+
+ CompactionStats majorCompact(MajorCompactionReason reason, long queued) {
+
+ CompactionStats majCStats = null;
+ boolean success = false;
+ long start = System.currentTimeMillis();
+
+ // Always trace majC
+ Span span = Trace.on("majorCompaction");
+
+ try {
+ timer.incrementStatusMajor();
+
+ synchronized (this) {
+ // check that compaction is still needed - defer to splitting
+ majorCompactionQueued.remove(reason);
+
+ if (isClosing() || isClosed() || !needsMajorCompaction(reason) || isMajorCompactionRunning() || needsSplit()) {
+ return null;
+ }
+
+ majorCompactionState = CompactionState.WAITING_TO_START;
+ }
+
+ try {
+ majCStats = _majorCompact(reason);
+ if (reason == MajorCompactionReason.CHOP) {
+ MetadataTableUtil.chopped(getExtent(), this.getTabletServer().getLock());
+ getTabletServer().enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.CHOPPED, extent));
+ }
+ success = true;
+ } catch (CompactionCanceledException mcce) {
+ log.debug("Major compaction canceled, extent = " + getExtent());
+ } catch (Throwable t) {
+ log.error("MajC Failed, extent = " + getExtent());
+ log.error("MajC Failed, message = " + (t.getMessage() == null ? t.getClass().getName() : t.getMessage()), t);
+ } finally {
+ // ensure we always reset boolean, even
+ // when an exception is thrown
+ synchronized (this) {
+ majorCompactionState = null;
+ this.notifyAll();
+ }
+
+ Span curr = Trace.currentTrace();
+ curr.data("extent", "" + getExtent());
+ if (majCStats != null) {
+ curr.data("read", "" + majCStats.getEntriesRead());
+ curr.data("written", "" + majCStats.getEntriesWritten());
+ }
+ }
+ } finally {
+ span.stop();
+ long count = 0;
+ if (majCStats != null)
+ count = majCStats.getEntriesRead();
+ timer.updateTime(Operation.MAJOR, queued, start, count, !success);
+ }
+
+ return majCStats;
+ }
+
+ /**
+ * Returns a KeyExtent object representing this tablet's key range.
+ *
+ * @return extent
+ */
+ @Override
+ public KeyExtent getExtent() {
+ return extent;
+ }
+
+ synchronized void computeNumEntries() {
+ Collection<DataFileValue> vals = getDatafileManager().getDatafileSizes().values();
+
+ long numEntries = 0;
+
+ for (DataFileValue tableValue : vals) {
+ numEntries += tableValue.getNumEntries();
+ }
+
+ this.numEntriesInMemory = getTabletMemory().getNumEntries();
+ numEntries += getTabletMemory().getNumEntries();
+
+ this.numEntries = numEntries;
+ }
+
+ public long getNumEntries() {
+ return numEntries;
+ }
+
+ public long getNumEntriesInMemory() {
+ return numEntriesInMemory;
+ }
+
+ public synchronized boolean isClosing() {
+ return closeState == CloseState.CLOSING;
+ }
+
+ public synchronized boolean isClosed() {
+ return closeState == CloseState.CLOSED || closeState == CloseState.COMPLETE;
+ }
+
+ public synchronized boolean isCloseComplete() {
+ return closeState == CloseState.COMPLETE;
+ }
+
+ public boolean isMajorCompactionRunning() {
+ return majorCompactionState != null;
+ }
+
+ public boolean isMinorCompactionQueued() {
+ return minorCompactionState == CompactionState.WAITING_TO_START;
+ }
+
+ public boolean isMinorCompactionRunning() {
+ return minorCompactionState == CompactionState.IN_PROGRESS;
+ }
+
+ public boolean isMajorCompactionQueued() {
+ return majorCompactionQueued.size() > 0;
+ }
+
+ public TreeMap<KeyExtent,SplitInfo> split(byte[] sp) throws IOException {
+
+ if (sp != null && extent.getEndRow() != null && extent.getEndRow().equals(new Text(sp))) {
+ throw new IllegalArgumentException();
+ }
+
+ if (sp != null && sp.length > tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE)) {
+ String msg = "Cannot split tablet " + extent + ", selected split point too long. Length : " + sp.length;
+ log.warn(msg);
+ throw new IOException(msg);
+ }
+
+ if (extent.isRootTablet()) {
+ String msg = "Cannot split root tablet";
+ log.warn(msg);
+ throw new RuntimeException(msg);
+ }
+
+ try {
+ initiateClose(true, false, false);
+ } catch (IllegalStateException ise) {
+ log.debug("File " + extent + " not splitting : " + ise.getMessage());
+ return null;
+ }
+
+ // obtain this info outside of synch block since it will involve opening
+ // the map files... it is ok if the set of map files changes, because
+ // this info is used for optimization... it is ok if map files are missing
+ // from the set... can still query and insert into the tablet while this
+ // map file operation is happening
+ Map<FileRef,FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(getTabletServer().getFileSystem(),
+ getTabletServer().getConfiguration(), getDatafileManager().getFiles());
+
+ synchronized (this) {
+ // java needs tuples ...
+ TreeMap<KeyExtent,SplitInfo> newTablets = new TreeMap<KeyExtent,SplitInfo>();
+
+ long t1 = System.currentTimeMillis();
+ // choose a split point
+ SplitRowSpec splitPoint;
+ if (sp == null)
+ splitPoint = findSplitRow(getDatafileManager().getFiles());
+ else {
+ Text tsp = new Text(sp);
+ splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(),
+ extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp), tsp);
+ }
+
+ if (splitPoint == null || splitPoint.row == null) {
+ log.info("had to abort split because splitRow was null");
+ closeState = CloseState.OPEN;
+ return null;
+ }
+
+ closeState = CloseState.CLOSING;
+ completeClose(true, false);
+
+ Text midRow = splitPoint.row;
+ double splitRatio = splitPoint.splitRatio;
+
+ KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow());
+ KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow);
+
+ String lowDirectory = TabletOperations.createTabletDirectory(getTabletServer().getFileSystem(), extent.getTableId().toString(), midRow);
+
+ // write new tablet information to MetadataTable
+ SortedMap<FileRef,DataFileValue> lowDatafileSizes = new TreeMap<FileRef,DataFileValue>();
+ SortedMap<FileRef,DataFileValue> highDatafileSizes = new TreeMap<FileRef,DataFileValue>();
+ List<FileRef> highDatafilesToRemove = new ArrayList<FileRef>();
+
+ MetadataTableUtil.splitDatafiles(extent.getTableId(), midRow, splitRatio, firstAndLastRows, getDatafileManager().getDatafileSizes(), lowDatafileSizes,
+ highDatafileSizes, highDatafilesToRemove);
+
+ log.debug("Files for low split " + low + " " + lowD
<TRUNCATED>
[34/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 71b7155,0000000..78f8c7b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -1,318 -1,0 +1,320 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tables;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.security.SecurityPermission;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.util.TablePropUtil;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+
+public class TableManager {
+ private static SecurityPermission TABLE_MANAGER_PERMISSION = new SecurityPermission("tableManagerPermission");
+
+ private static final Logger log = Logger.getLogger(TableManager.class);
+ private static final Set<TableObserver> observers = Collections.synchronizedSet(new HashSet<TableObserver>());
+ private static final Map<String,TableState> tableStateCache = Collections.synchronizedMap(new HashMap<String,TableState>());
+ private static final byte[] ZERO_BYTE = new byte[] {'0'};
+
+ private static TableManager tableManager = null;
+
+ private final Instance instance;
+ private ZooCache zooStateCache;
+
+ public static void prepareNewNamespaceState(String instanceId, String namespaceId, String namespace, NodeExistsPolicy existsPolicy) throws KeeperException,
+ InterruptedException {
+ log.debug("Creating ZooKeeper entries for new namespace " + namespace + " (ID: " + namespaceId + ")");
+ String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId;
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zPath, new byte[0], existsPolicy);
- zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(UTF_8), existsPolicy);
+ zoo.putPersistentData(zPath + Constants.ZNAMESPACE_CONF, new byte[0], existsPolicy);
+ }
+
+ public static void prepareNewTableState(String instanceId, String tableId, String namespaceId, String tableName, TableState state,
+ NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ // state gets created last
+ log.debug("Creating ZooKeeper entries for new table " + tableName + " (ID: " + tableId + ") in namespace (ID: " + namespaceId + ")");
+ Pair<String,String> qualifiedTableName = Tables.qualify(tableName);
+ tableName = qualifiedTableName.getSecond();
+ String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId;
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ zoo.putPersistentData(zTablePath, new byte[0], existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_CONF, new byte[0], existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(Constants.UTF8), existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.getBytes(UTF_8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(UTF_8), existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE, existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE, existsPolicy);
+ zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID, ZERO_BYTE, existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(Constants.UTF8), existsPolicy);
++ zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(UTF_8), existsPolicy);
+ }
+
+ public synchronized static TableManager getInstance() {
+ SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPermission(TABLE_MANAGER_PERMISSION);
+ }
+ if (tableManager == null)
+ tableManager = new TableManager();
+ return tableManager;
+ }
+
+ private TableManager() {
+ instance = HdfsZooInstance.getInstance();
+ zooStateCache = new ZooCache(new TableStateWatcher());
+ updateTableStateCache();
+ }
+
+ public TableState getTableState(String tableId) {
+ return tableStateCache.get(tableId);
+ }
+
+ public static class IllegalTableTransitionException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ final TableState oldState;
+ final TableState newState;
+
+ public IllegalTableTransitionException(TableState oldState, TableState newState) {
+ this.oldState = oldState;
+ this.newState = newState;
+ }
+
+ public TableState getOldState() {
+ return oldState;
+ }
+
+ public TableState getNewState() {
+ return newState;
+ }
+
+ }
+
+ public synchronized void transitionTableState(final String tableId, final TableState newState) {
+ String statePath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE;
+
+ try {
- ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(Constants.UTF8), ZooUtil.PUBLIC, new Mutator() {
++ ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(UTF_8), ZooUtil.PUBLIC, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] oldData) throws Exception {
+ TableState oldState = TableState.UNKNOWN;
+ if (oldData != null)
- oldState = TableState.valueOf(new String(oldData, Constants.UTF8));
++ oldState = TableState.valueOf(new String(oldData, UTF_8));
+ boolean transition = true;
+ // +--------+
+ // v |
+ // NEW -> (ONLINE|OFFLINE)+--- DELETING
+ switch (oldState) {
+ case NEW:
+ transition = (newState == TableState.OFFLINE || newState == TableState.ONLINE);
+ break;
+ case ONLINE: // fall-through intended
+ case UNKNOWN:// fall through intended
+ case OFFLINE:
+ transition = (newState != TableState.NEW);
+ break;
+ case DELETING:
+ // Can't transition to any state from DELETING
+ transition = false;
+ break;
+ }
+ if (!transition)
+ throw new IllegalTableTransitionException(oldState, newState);
+ log.debug("Transitioning state for table " + tableId + " from " + oldState + " to " + newState);
- return newState.name().getBytes(Constants.UTF8);
++ return newState.name().getBytes(UTF_8);
+ }
+ });
+ } catch (Exception e) {
+ log.fatal("Failed to transition table to state " + newState);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void updateTableStateCache() {
+ synchronized (tableStateCache) {
+ for (String tableId : zooStateCache.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES))
+ if (zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE) != null)
+ updateTableStateCache(tableId);
+ }
+ }
+
+ public TableState updateTableStateCache(String tableId) {
+ synchronized (tableStateCache) {
+ TableState tState = TableState.UNKNOWN;
+ byte[] data = zooStateCache.get(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE);
+ if (data != null) {
- String sState = new String(data, Constants.UTF8);
++ String sState = new String(data, UTF_8);
+ try {
+ tState = TableState.valueOf(sState);
+ } catch (IllegalArgumentException e) {
+ log.error("Unrecognized state for table with tableId=" + tableId + ": " + sState);
+ }
+ tableStateCache.put(tableId, tState);
+ }
+ return tState;
+ }
+ }
+
+ public void addTable(String tableId, String namespaceId, String tableName, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException,
+ NamespaceNotFoundException {
+ prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
+ updateTableStateCache(tableId);
+ }
+
+ public void cloneTable(String srcTable, String tableId, String tableName, String namespaceId, Map<String,String> propertiesToSet,
+ Set<String> propertiesToExclude, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException {
+ prepareNewTableState(instance.getInstanceID(), tableId, namespaceId, tableName, TableState.NEW, existsPolicy);
+
+ String srcTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + srcTable + Constants.ZTABLE_CONF;
+ String newTablePath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF;
+ ZooReaderWriter.getInstance().recursiveCopyPersistent(srcTablePath, newTablePath, NodeExistsPolicy.OVERWRITE);
+
+ for (Entry<String,String> entry : propertiesToSet.entrySet())
+ TablePropUtil.setTableProperty(tableId, entry.getKey(), entry.getValue());
+
+ for (String prop : propertiesToExclude)
+ ZooReaderWriter.getInstance().recursiveDelete(
+ Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_CONF + "/" + prop, NodeMissingPolicy.SKIP);
+
+ updateTableStateCache(tableId);
+ }
+
+ public void removeTable(String tableId) throws KeeperException, InterruptedException {
+ synchronized (tableStateCache) {
+ tableStateCache.remove(tableId);
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE,
+ NodeMissingPolicy.SKIP);
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId, NodeMissingPolicy.SKIP);
+ }
+ }
+
+ public boolean addObserver(TableObserver to) {
+ synchronized (observers) {
+ synchronized (tableStateCache) {
+ to.initialize(Collections.unmodifiableMap(tableStateCache));
+ return observers.add(to);
+ }
+ }
+ }
+
+ public boolean removeObserver(TableObserver to) {
+ return observers.remove(to);
+ }
+
+ private class TableStateWatcher implements Watcher {
+ @Override
+ public void process(WatchedEvent event) {
+ if (log.isTraceEnabled())
+ log.trace(event);
+
+ final String zPath = event.getPath();
+ final EventType zType = event.getType();
+
+ String tablesPrefix = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ String tableId = null;
+
+ if (zPath != null && zPath.startsWith(tablesPrefix + "/")) {
+ String suffix = zPath.substring(tablesPrefix.length() + 1);
+ if (suffix.contains("/")) {
+ String[] sa = suffix.split("/", 2);
+ if (Constants.ZTABLE_STATE.equals("/" + sa[1]))
+ tableId = sa[0];
+ }
+ if (tableId == null) {
+ log.warn("Unknown path in " + event);
+ return;
+ }
+ }
+
+ switch (zType) {
+ case NodeChildrenChanged:
+ if (zPath != null && zPath.equals(tablesPrefix)) {
+ updateTableStateCache();
+ } else {
+ log.warn("Unexpected path " + zPath);
+ }
+ break;
+ case NodeCreated:
+ case NodeDataChanged:
+ // state transition
+ TableState tState = updateTableStateCache(tableId);
+ log.debug("State transition to " + tState + " @ " + event);
+ synchronized (observers) {
+ for (TableObserver to : observers)
+ to.stateChanged(tableId, tState);
+ }
+ break;
+ case NodeDeleted:
+ if (zPath != null
+ && tableId != null
+ && (zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_STATE) || zPath.equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_CONF) || zPath
+ .equals(tablesPrefix + "/" + tableId + Constants.ZTABLE_NAME)))
+ tableStateCache.remove(tableId);
+ break;
+ case None:
+ switch (event.getState()) {
+ case Expired:
+ if (log.isTraceEnabled())
+ log.trace("Session expired " + event);
+ synchronized (observers) {
+ for (TableObserver to : observers)
+ to.sessionExpired();
+ }
+ break;
+ case SyncConnected:
+ default:
+ if (log.isTraceEnabled())
+ log.trace("Ignored " + event);
+ }
+ break;
+ default:
+ log.warn("Unandled " + event);
+ }
+ }
+ }
+
+ public void removeNamespace(String namespaceId) throws KeeperException, InterruptedException {
+ ZooReaderWriter.getInstance().recursiveDelete(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId, NodeMissingPolicy.SKIP);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index eabdc0d,0000000..e10c0c1
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -1,79 -1,0 +1,81 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tablets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Random;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+/**
+ * Allocates unique names for an accumulo instance. The names are unique for the lifetime of the instance.
+ *
+ * This is useful for filenames because it makes caching easy.
+ *
+ */
+
+public class UniqueNameAllocator {
+ private long next = 0;
+ private long maxAllocated = 0;
+ private String nextNamePath;
+ private Random rand;
+
+ private UniqueNameAllocator() {
+ nextNamePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZNEXT_FILE;
+ rand = new Random();
+ }
+
+ public synchronized String getNextName() {
+
+ while (next >= maxAllocated) {
+ final int allocate = 100 + rand.nextInt(100);
+
+ try {
+ byte[] max = ZooReaderWriter.getInstance().mutate(nextNamePath, null, ZooUtil.PRIVATE, new ZooReaderWriter.Mutator() {
+ public byte[] mutate(byte[] currentValue) throws Exception {
- long l = Long.parseLong(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++ long l = Long.parseLong(new String(currentValue, UTF_8), Character.MAX_RADIX);
+ l += allocate;
- return Long.toString(l, Character.MAX_RADIX).getBytes(Constants.UTF8);
++ return Long.toString(l, Character.MAX_RADIX).getBytes(UTF_8);
+ }
+ });
+
- maxAllocated = Long.parseLong(new String(max, Constants.UTF8), Character.MAX_RADIX);
++ maxAllocated = Long.parseLong(new String(max, UTF_8), Character.MAX_RADIX);
+ next = maxAllocated - allocate;
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
- return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), Constants.UTF8);
++ return new String(FastFormat.toZeroPaddedString(next++, 7, Character.MAX_RADIX, new byte[0]), UTF_8);
+ }
+
+ private static UniqueNameAllocator instance = null;
+
+ public static synchronized UniqueNameAllocator getInstance() {
+ if (instance == null)
+ instance = new UniqueNameAllocator();
+
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index cdff21d,0000000..198b2d6
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -1,136 -1,0 +1,137 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class AddFilesWithMissingEntries {
+
+ static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
+
+ public static class Opts extends ClientOpts {
+ @Parameter(names = "-update", description = "Make changes to the " + MetadataTable.NAME + " table to include missing files")
+ boolean update = false;
+ }
+
+ /**
+ * A utility to add files to the {@value MetadataTable#NAME} table that are not listed in the root tablet. This is a recovery tool for someone who knows what
+ * they are doing. It might be better to save off files, and recover your instance by re-initializing and importing the existing files.
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(AddFilesWithMissingEntries.class.getName(), args, bwOpts);
+
+ final Scanner scanner = opts.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ final Configuration conf = new Configuration();
+ final FileSystem fs = FileSystem.get(conf);
+
+ KeyExtent last = new KeyExtent();
+ String directory = null;
+ Set<String> knownFiles = new HashSet<String>();
+
+ int count = 0;
+ final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
+
+ // collect the list of known files and the directory for each extent
+ for (Entry<Key,Value> entry : scanner) {
+ Key key = entry.getKey();
+ KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
+ // when the key extent changes
+ if (!ke.equals(last)) {
+ if (directory != null) {
+ // add any files in the directory unknown to the key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ directory = null;
+ knownFiles.clear();
+ last = ke;
+ }
+ if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
+ directory = entry.getValue().toString();
+ log.debug("Found directory " + directory + " for row " + key.getRow().toString());
+ } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
+ String filename = key.getColumnQualifier().toString();
+ knownFiles.add(filename);
+ log.debug("METADATA file found: " + filename);
+ }
+ }
+ if (directory != null) {
+ // catch the last key extent
+ count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
+ }
+ log.info("There were " + count + " files that are unknown to the metadata table");
+ writer.close();
+ }
+
+ private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update)
+ throws Exception {
+ int count = 0;
+ final String tableId = ke.getTableId().toString();
+ final Text row = ke.getMetadataEntry();
+ log.info(row.toString());
+ for (String dir : ServerConstants.getTablesDirs()) {
+ final Path path = new Path(dir + "/" + tableId + directory);
+ for (FileStatus file : fs.listStatus(path)) {
+ if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf"))
+ continue;
+ final String filename = directory + "/" + file.getPath().getName();
+ if (!knownFiles.contains(filename)) {
+ count++;
+ final Mutation m = new Mutation(row);
+ String size = Long.toString(file.getLen());
+ String entries = "1"; // lie
+ String value = size + "," + entries;
- m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(Constants.UTF8)));
++ m.put(DataFileColumnFamily.NAME, new Text(filename), new Value(value.getBytes(UTF_8)));
+ if (update) {
+ writer.getBatchWriter(MetadataTable.NAME).addMutation(m);
+ }
+ }
+ }
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 2926a3f,0000000..7a1e1ba
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@@ -1,166 -1,0 +1,167 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class ChangeSecret {
+
+ static class Opts extends ClientOpts {
+ @Parameter(names="--old", description="old zookeeper password", password=true, hidden=true)
+ String oldPass;
+ @Parameter(names="--new", description="new zookeeper password", password=true, hidden=true)
+ String newPass;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ List<String> argsList = new ArrayList<String>(args.length + 2);
+ argsList.add("--old");
+ argsList.add("--new");
+ argsList.addAll(Arrays.asList(args));
+ opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
+ VolumeManager fs = VolumeManagerImpl.get();
+ Instance inst = opts.getInstance();
+ if (!verifyAccumuloIsDown(inst, opts.oldPass))
+ System.exit(-1);
+ String instanceId = rewriteZooKeeperInstance(inst, opts.oldPass, opts.newPass);
+ updateHdfs(fs, inst, instanceId);
+ if (opts.oldPass != null) {
+ deleteInstance(inst, opts.oldPass);
+ }
+ System.out.println("New instance id is " + instanceId);
+ System.out.println("Be sure to put your new secret in accumulo-site.xml");
+ }
+
+ interface Visitor {
+ void visit(ZooReader zoo, String path) throws Exception;
+ }
+
+ private static void recurse(ZooReader zoo, String root, Visitor v) {
+ try {
+ v.visit(zoo, root);
+ for (String child : zoo.getChildren(root)) {
+ recurse(zoo, root + "/" + child, v);
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private static boolean verifyAccumuloIsDown(Instance inst, String oldPassword) {
+ ZooReader zooReader = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPassword);
+ String root = ZooUtil.getRoot(inst);
+ final List<String> ephemerals = new ArrayList<String>();
+ recurse(zooReader, root, new Visitor() {
+ public void visit(ZooReader zoo, String path) throws Exception {
+ Stat stat = zoo.getStatus(path);
+ if (stat.getEphemeralOwner() != 0)
+ ephemerals.add(path);
+ }
+ });
+ if (ephemerals.size() == 0) {
+ return true;
+ }
+
+ System.err.println("The following ephemeral nodes exist, something is still running:");
+ for (String path : ephemerals) {
+ System.err.println(path);
+ }
+ return false;
+ }
+
+ private static String rewriteZooKeeperInstance(final Instance inst, String oldPass, String newPass) throws Exception {
+ final ZooReaderWriter orig = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), oldPass);
+ final IZooReaderWriter new_ = new ZooReaderWriter(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), newPass);
+ final String newInstanceId = UUID.randomUUID().toString();
+ String root = ZooUtil.getRoot(inst);
+ recurse(orig, root, new Visitor() {
+ public void visit(ZooReader zoo, String path) throws Exception {
+ String newPath = path.replace(inst.getInstanceID(), newInstanceId);
+ byte[] data = zoo.getData(path, null);
+ List<ACL> acls = orig.getZooKeeper().getACL(path, new Stat());
+ if (acls.containsAll(Ids.READ_ACL_UNSAFE)) {
+ new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ } else {
+ // upgrade
+ if (acls.containsAll(Ids.OPEN_ACL_UNSAFE)) {
+ // make user nodes private, they contain the user's password
+ String parts[] = path.split("/");
+ if (parts[parts.length - 2].equals("users")) {
+ new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ } else {
+ // everything else can have the readable acl
+ new_.putPersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ }
+ } else {
+ new_.putPrivatePersistentData(newPath, data, NodeExistsPolicy.FAIL);
+ }
+ }
+ }
+ });
+ String path = "/accumulo/instances/" + inst.getInstanceName();
+ orig.recursiveDelete(path, NodeMissingPolicy.SKIP);
- new_.putPersistentData(path, newInstanceId.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ new_.putPersistentData(path, newInstanceId.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ return newInstanceId;
+ }
+
+ private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
+ // Need to recreate the instanceId on all of them to keep consistency
+ for (Volume v : fs.getVolumes()) {
+ final Path instanceId = ServerConstants.getInstanceIdLocation(v);
+ if (!v.getFileSystem().delete(instanceId, true)) {
+ throw new IOException("Could not recursively delete " + instanceId);
+ }
+
+ if (!v.getFileSystem().mkdirs(instanceId)) {
+ throw new IOException("Could not create directory " + instanceId);
+ }
+
+ v.getFileSystem().create(new Path(instanceId, newInstanceId)).close();
+ }
+ }
+
+ private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
+ IZooReaderWriter orig = new ZooReaderWriter(origInstance.getZooKeepers(), origInstance.getZooKeepersSessionTimeOut(), oldPass);
+ orig.recursiveDelete("/accumulo/" + origInstance.getInstanceID(), NodeMissingPolicy.SKIP);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
index f2074a1,0000000..809a7c8
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/CleanZookeeper.java
@@@ -1,87 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class CleanZookeeper {
+
+ private static final Logger log = Logger.getLogger(CleanZookeeper.class);
+
+ static class Opts extends Help {
+ @Parameter(names={"-z", "--keepers"}, description="comma separated list of zookeeper hosts")
+ String keepers = "localhost:2181";
+ @Parameter(names={"--password"}, description="the system secret", password=true)
+ String auth;
+ }
+
+ /**
+ * @param args
+ * must contain one element: the address of a zookeeper node a second parameter provides an additional authentication value
+ * @throws IOException
+ * error connecting to accumulo or zookeeper
+ */
+ public static void main(String[] args) throws IOException {
+ Opts opts = new Opts();
+ opts.parseArgs(CleanZookeeper.class.getName(), args);
+
+ String root = Constants.ZROOT;
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (opts.auth != null) {
- zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(Constants.UTF8));
++ zk.getZooKeeper().addAuthInfo("digest", ("accumulo:"+opts.auth).getBytes(UTF_8));
+ }
+
+ try {
+ for (String child : zk.getChildren(root)) {
+ if (Constants.ZINSTANCES.equals("/" + child)) {
+ for (String instanceName : zk.getChildren(root + Constants.ZINSTANCES)) {
+ String instanceNamePath = root + Constants.ZINSTANCES + "/" + instanceName;
+ byte[] id = zk.getData(instanceNamePath, null);
- if (id != null && !new String(id, Constants.UTF8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
++ if (id != null && !new String(id, UTF_8).equals(HdfsZooInstance.getInstance().getInstanceID())) {
+ try {
+ zk.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
+ } catch (KeeperException.NoAuthException ex) {
+ log.warn("Unable to delete " + instanceNamePath);
+ }
+ }
+ }
+ } else if (!child.equals(HdfsZooInstance.getInstance().getInstanceID())) {
+ String path = root + "/" + child;
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ } catch (KeeperException.NoAuthException ex) {
+ log.warn("Unable to delete " + path);
+ }
+ }
+ }
+ } catch (Exception ex) {
+ System.out.println("Error Occurred: " + ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 448da86,0000000..d40830f
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@@ -1,81 -1,0 +1,83 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+import com.beust.jcommander.Parameter;
+
+public class DeleteZooInstance {
+
+ static class Opts extends Help {
+ @Parameter(names={"-i", "--instance"}, description="the instance name or id to delete")
+ String instance;
+ }
+
+ static void deleteRetry(IZooReaderWriter zk, String path) throws Exception {
+ for (int i = 0; i < 10; i++){
+ try {
+ zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
+ return;
+ } catch (KeeperException.NotEmptyException ex) {
+ // ignored
+ } catch (Exception ex) {
+ throw ex;
+ }
+ }
+ }
+
+ /**
+ * @param args
+ * : the name or UUID of the instance to be deleted
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(DeleteZooInstance.class.getName(), args);
+
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ // try instance name:
+ Set<String> instances = new HashSet<String>(zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES));
+ Set<String> uuids = new HashSet<String>(zk.getChildren(Constants.ZROOT));
+ uuids.remove("instances");
+ if (instances.contains(opts.instance)) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + opts.instance;
+ byte[] data = zk.getData(path, null);
+ deleteRetry(zk, path);
- deleteRetry(zk, Constants.ZROOT + "/" + new String(data, Constants.UTF8));
++ deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
+ } else if (uuids.contains(opts.instance)) {
+ // look for the real instance name
+ for (String instance : instances) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
+ byte[] data = zk.getData(path, null);
- if (opts.instance.equals(new String(data, Constants.UTF8)))
++ if (opts.instance.equals(new String(data, UTF_8)))
+ deleteRetry(zk, path);
+ }
+ deleteRetry(zk, Constants.ZROOT + "/" + opts.instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 504956f,0000000..43348df
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@@ -1,121 -1,0 +1,122 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import com.beust.jcommander.Parameter;
+
+public class DumpZookeeper {
+
+ static IZooReaderWriter zk = null;
+
+ private static final Logger log = Logger.getLogger(DumpZookeeper.class);
+
+ private static class Encoded {
+ public String encoding;
+ public String value;
+
+ Encoded(String e, String v) {
+ encoding = e;
+ value = v;
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = "--root", description = "the root of the znode tree to dump")
+ String root = "/";
+ }
+
+ public static void main(String[] args) {
+ Opts opts = new Opts();
+ opts.parseArgs(DumpZookeeper.class.getName(), args);
+
+ Logger.getRootLogger().setLevel(Level.WARN);
+ PrintStream out = System.out;
+ try {
+ zk = ZooReaderWriter.getInstance();
+
+ write(out, 0, "<dump root='%s'>", opts.root);
+ for (String child : zk.getChildren(opts.root, null))
+ if (!child.equals("zookeeper"))
+ dump(out, opts.root, child, 1);
+ write(out, 0, "</dump>");
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ private static void dump(PrintStream out, String root, String child, int indent) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ String path = root + "/" + child;
+ if (root.endsWith("/"))
+ path = root + child;
+ Stat stat = zk.getStatus(path);
+ if (stat == null)
+ return;
+ String type = "node";
+ if (stat.getEphemeralOwner() != 0) {
+ type = "ephemeral";
+ }
+ if (stat.getNumChildren() == 0) {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'/>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'/>", type, child, value.encoding, value.value);
+ }
+ } else {
+ if (stat.getDataLength() == 0) {
+ write(out, indent, "<%s name='%s'>", type, child);
+ } else {
+ Encoded value = value(path);
+ write(out, indent, "<%s name='%s' encoding='%s' value='%s'>", type, child, value.encoding, value.value);
+ }
+ for (String c : zk.getChildren(path, null)) {
+ dump(out, path, c, indent + 1);
+ }
+ write(out, indent, "</node>");
+ }
+ }
+
+ private static Encoded value(String path) throws KeeperException, InterruptedException, UnsupportedEncodingException {
+ byte[] data = zk.getData(path, null);
+ for (int i = 0; i < data.length; i++) {
+ // does this look like simple ascii?
+ if (data[i] < ' ' || data[i] > '~')
+ return new Encoded("base64", Base64.encodeBase64String(data));
+ }
- return new Encoded(Constants.UTF8.name(), new String(data, Constants.UTF8));
++ return new Encoded(UTF_8.name(), new String(data, UTF_8));
+ }
+
+ private static void write(PrintStream out, int indent, String fmt, Object... args) {
+ for (int i = 0; i < indent; i++)
+ out.print(" ");
+ out.println(String.format(fmt, args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
index f4655dc,0000000..be85b1b
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java
@@@ -1,225 -1,0 +1,227 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Formattable;
+import java.util.Formatter;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class ListInstances {
+
+ private static final Logger log = Logger.getLogger(ListInstances.class);
+
+ private static final int NAME_WIDTH = 20;
+ private static final int UUID_WIDTH = 37;
+ private static final int MASTER_WIDTH = 30;
+
+ private static final int ZOOKEEPER_TIMER_MILLIS = 30 * 1000;
+
+ static class Opts extends Help {
+ @Parameter(names="--print-errors", description="display errors while listing instances")
+ boolean printErrors = false;
+ @Parameter(names="--print-all", description="print information for all instances, not just those with names")
+ boolean printAll = false;
+ @Parameter(names={"-z", "--zookeepers"}, description="the zookeepers to contact")
+ String keepers = null;
+ }
+ static Opts opts = new Opts();
+ static int errors = 0;
+
+ public static void main(String[] args) {
+ opts.parseArgs(ListInstances.class.getName(), args);
+
+ if (opts.keepers == null) {
+ opts.keepers = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
+ }
+
+ String keepers = opts.keepers;
+ boolean printAll = opts.printAll;
+ boolean printErrors = opts.printErrors;
+
+ listInstances(keepers, printAll, printErrors);
+
+ }
+
+ static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) {
+ errors = 0;
+
+ System.out.println("INFO : Using ZooKeepers " + keepers);
+ ZooReader rdr = new ZooReader(keepers, ZOOKEEPER_TIMER_MILLIS);
+ ZooCache cache = new ZooCache(keepers, ZOOKEEPER_TIMER_MILLIS);
+
+ TreeMap<String,UUID> instanceNames = getInstanceNames(rdr, printErrors);
+
+ System.out.println();
+ printHeader();
+
+ for (Entry<String,UUID> entry : instanceNames.entrySet()) {
+ printInstanceInfo(cache, entry.getKey(), entry.getValue(), printErrors);
+ }
+
+ TreeSet<UUID> instancedIds = getInstanceIDs(rdr, printErrors);
+ instancedIds.removeAll(instanceNames.values());
+
+ if (printAll) {
+ for (UUID uuid : instancedIds) {
+ printInstanceInfo(cache, null, uuid, printErrors);
+ }
+ } else if (instancedIds.size() > 0) {
+ System.out.println();
+ System.out.println("INFO : " + instancedIds.size() + " unamed instances were not printed, run with --print-all to see all instances");
+ } else {
+ System.out.println();
+ }
+
+ if (!printErrors && errors > 0) {
+ System.err.println("WARN : There were " + errors + " errors, run with --print-errors to see more info");
+ }
+ }
+
+ private static class CharFiller implements Formattable {
+
+ char c;
+
+ CharFiller(char c) {
+ this.c = c;
+ }
+
+ @Override
+ public void formatTo(Formatter formatter, int flags, int width, int precision) {
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < width; i++)
+ sb.append(c);
+ formatter.format(sb.toString());
+ }
+
+ }
+
+ private static void printHeader() {
+ System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MASTER_WIDTH + "s%n", "Instance Name", "Instance ID", "Master");
+ System.out.printf("%" + (NAME_WIDTH + 1) + "s+%" + (UUID_WIDTH + 1) + "s+%" + (MASTER_WIDTH + 1) + "s%n", new CharFiller('-'), new CharFiller('-'),
+ new CharFiller('-'));
+
+ }
+
+ private static void printInstanceInfo(ZooCache cache, String instanceName, UUID iid, boolean printErrors) {
+ String master = getMaster(cache, iid, printErrors);
+ if (instanceName == null) {
+ instanceName = "";
+ }
+
+ if (master == null) {
+ master = "";
+ }
+
+ System.out.printf("%" + NAME_WIDTH + "s |%" + UUID_WIDTH + "s |%" + MASTER_WIDTH + "s%n", "\"" + instanceName + "\"", iid, master);
+ }
+
+ private static String getMaster(ZooCache cache, UUID iid, boolean printErrors) {
+
+ if (iid == null) {
+ return null;
+ }
+
+ try {
+ String masterLocPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK;
+ byte[] master = ZooLock.getLockData(cache, masterLocPath, null);
+ if (master == null) {
+ return null;
+ }
- return new String(master, Constants.UTF8);
++ return new String(master, UTF_8);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ return null;
+ }
+ }
+
+ private static TreeMap<String,UUID> getInstanceNames(ZooReader zk, boolean printErrors) {
+
+ String instancesPath = Constants.ZROOT + Constants.ZINSTANCES;
+
+ TreeMap<String,UUID> tm = new TreeMap<String,UUID>();
+
+ List<String> names;
+
+ try {
+ names = zk.getChildren(instancesPath);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ return tm;
+ }
+
+ for (String name : names) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ try {
- UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), Constants.UTF8));
++ UUID iid = UUID.fromString(new String(zk.getData(instanceNamePath, null), UTF_8));
+ tm.put(name, iid);
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ tm.put(name, null);
+ }
+ }
+
+ return tm;
+ }
+
+ private static TreeSet<UUID> getInstanceIDs(ZooReader zk, boolean printErrors) {
+ TreeSet<UUID> ts = new TreeSet<UUID>();
+
+ try {
+ List<String> children = zk.getChildren(Constants.ZROOT);
+
+ for (String iid : children) {
+ if (iid.equals("instances"))
+ continue;
+ try {
+ ts.add(UUID.fromString(iid));
+ } catch (Exception e) {
+ log.error("Exception: " + e);
+ }
+ }
+ } catch (Exception e) {
+ handleException(e, printErrors);
+ }
+
+ return ts;
+ }
+
+ private static void handleException(Exception e, boolean printErrors) {
+ if (printErrors) {
+ e.printStackTrace();
+ }
+
+ errors++;
+ }
+}
[02/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a338d5f,0000000..d5c940d
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@@ -1,553 -1,0 +1,553 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.DiskUsage;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class VolumeIT extends ConfigurableMacIT {
+
+ private static final Text EMPTY = new Text();
+ private static final Value EMPTY_VALUE = new Value(new byte[] {});
+ private File volDirBase;
+ private Path v1, v2;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ File baseDir = cfg.getDir();
+ volDirBase = new File(baseDir, "volumes");
+ File v1f = new File(volDirBase, "v1");
+ File v2f = new File(volDirBase, "v2");
+ v1f.mkdir();
+ v2f.mkdir();
+ v1 = new Path("file://" + v1f.getAbsolutePath());
+ v2 = new Path("file://" + v2f.getAbsolutePath());
+
+ // Run MAC on two locations in the local file system
+ URI v1Uri = v1.toUri();
+ cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
+ cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
+ cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+
+ super.configure(cfg, hadoopCoreSite);
+ }
+
+ @Test
+ public void test() throws Exception {
+ // create a table
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName);
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ // with some splits
+ for (String s : "d,m,t".split(","))
+ partitions.add(new Text(s));
+ connector.tableOperations().addSplits(tableName, partitions);
+ // scribble over the splits
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+ String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ for (String s : rows) {
+ Mutation m = new Mutation(new Text(s));
+ m.put(EMPTY, EMPTY, EMPTY_VALUE);
+ bw.addMutation(m);
+ }
+ bw.close();
+ // write the data to disk, read it back
+ connector.tableOperations().flush(tableName, null, null, true);
+ Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
+ int i = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ assertEquals(rows[i++], entry.getKey().getRow().toString());
+ }
+ // verify the new files are written to the different volumes
+ scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(new Range("1", "1<"));
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ int fileCount = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ boolean inV1 = entry.getKey().getColumnQualifier().toString().contains(v1.toString());
+ boolean inV2 = entry.getKey().getColumnQualifier().toString().contains(v2.toString());
+ assertTrue(inV1 || inV2);
+ fileCount++;
+ }
+ assertEquals(4, fileCount);
+ List<DiskUsage> diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
+ assertEquals(1, diskUsage.size());
+ long usage = diskUsage.get(0).getUsage().longValue();
+ System.out.println("usage " + usage);
+ assertTrue(usage > 700 && usage < 800);
+ }
+
+ private void verifyData(List<String> expected, Scanner createScanner) {
+
+ List<String> actual = new ArrayList<String>();
+
+ for (Entry<Key,Value> entry : createScanner) {
+ Key k = entry.getKey();
+ actual.add(k.getRow() + ":" + k.getColumnFamily() + ":" + k.getColumnQualifier() + ":" + entry.getValue());
+ }
+
+ Collections.sort(expected);
+ Collections.sort(actual);
+
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testRelativePaths() throws Exception {
+
+ List<String> expected = new ArrayList<String>();
+
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName, false);
+
+ String tableId = connector.tableOperations().tableIdMap().get(tableName);
+
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ // with some splits
+ for (String s : "c,g,k,p,s,v".split(","))
+ partitions.add(new Text(s));
+
+ connector.tableOperations().addSplits(tableName, partitions);
+
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+
+ // create two files in each tablet
+
+ String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ for (String s : rows) {
+ Mutation m = new Mutation(s);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ expected.add(s + ":cf1:cq1:1");
+ }
+
+ bw.flush();
+ connector.tableOperations().flush(tableName, null, null, true);
+
+ for (String s : rows) {
+ Mutation m = new Mutation(s);
+ m.put("cf1", "cq1", "2");
+ bw.addMutation(m);
+ expected.add(s + ":cf1:cq1:2");
+ }
+
+ bw.close();
+ connector.tableOperations().flush(tableName, null, null, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ connector.tableOperations().offline(tableName, true);
+
+ connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+
+ Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ String cq = entry.getKey().getColumnQualifier().toString();
+ if (cq.startsWith(v1.toString())) {
+ Path path = new Path(cq);
+ String relPath = "/" + path.getParent().getName() + "/" + path.getName();
+ Mutation fileMut = new Mutation(entry.getKey().getRow());
+ fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
+ mbw.addMutation(fileMut);
+ }
+ }
+
+ mbw.close();
+
+ connector.tableOperations().online(tableName, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ connector.tableOperations().compact(tableName, null, null, true, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ String cq = entry.getKey().getColumnQualifier().toString();
+ Path path = new Path(cq);
+ Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
+ }
+
+ }
+
+ @Test
+ public void testAddVolumes() throws Exception {
+
+ String[] tableNames = getUniqueNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v1.toString() + "," + v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ verifyVolumesUsed(tableNames[1], false, v1, v2, v3);
+ }
+
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
+
+ String[] tableNames = getUniqueNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ v3f.mkdir();
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
+
+ // v1 should not have any data for tableNames[1]
+ verifyVolumesUsed(tableNames[1], false, v2, v3);
+ }
+
+ private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
+ MutationsRejectedException {
+ TreeSet<Text> splits = new TreeSet<Text>();
+ for (int i = 1; i < 100; i++) {
+ splits.add(new Text(String.format("%06d", i * 100)));
+ }
+
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().addSplits(tableName, splits);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ Mutation m = new Mutation(row);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+
+ private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws AccumuloException, AccumuloSecurityException,
+ TableExistsException, TableNotFoundException, MutationsRejectedException {
+
+ Connector conn = getConnector();
+
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ if (!conn.tableOperations().exists(tableName)) {
+ Assert.assertFalse(shouldExist);
+
+ writeData(tableName, conn);
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ conn.tableOperations().flush(tableName, null, null, true);
+ }
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
+ metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ int counts[] = new int[paths.length];
+
+ outer: for (Entry<Key,Value> entry : metaScanner) {
+ String cf = entry.getKey().getColumnFamily().toString();
+ String cq = entry.getKey().getColumnQualifier().toString();
+
+ String path;
+ if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
+ path = cq;
+ else
+ path = entry.getValue().toString();
+
+ for (int i = 0; i < paths.length; i++) {
+ if (path.startsWith(paths[i].toString())) {
+ counts[i]++;
+ continue outer;
+ }
+ }
+
+ Assert.fail("Unexpected volume " + path);
+ }
+
+ // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
+ // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
+
+ int sum = 0;
+ for (int count : counts) {
+ Assert.assertTrue(count > 0);
+ sum += count;
+ }
+
+ Assert.assertEquals(200, sum);
+ }
+
+ @Test
+ public void testRemoveVolumes() throws Exception {
+ String[] tableNames = getUniqueNames(2);
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volume was decommisioned
+ cluster.start();
+
+ Connector conn = cluster.getConnector("root", ROOT_PASSWORD);
+ conn.tableOperations().compact(tableNames[0], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], true, v2);
+
+ // check that root tablet is not on volume 1
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
+ ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
- String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++ String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
+ Assert.assertTrue(rootTabletDir.startsWith(v2.toString()));
+ zookeeper.close();
+
+ conn.tableOperations().clone(tableNames[0], tableNames[1], true, new HashMap<String,String>(), new HashSet<String>());
+
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ conn.tableOperations().flush(RootTable.NAME, null, null, true);
+
+ verifyVolumesUsed(tableNames[0], true, v2);
+ verifyVolumesUsed(tableNames[1], true, v2);
+
+ }
+
+ private void testReplaceVolume(boolean cleanShutdown) throws Exception {
+ String[] tableNames = getUniqueNames(3);
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ // write to 2nd table, but do not flush data to disk before shutdown
+ writeData(tableNames[1], cluster.getConnector("root", ROOT_PASSWORD));
+
+ if (cleanShutdown)
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+
+ cluster.stop();
+
+ File v1f = new File(v1.toUri());
+ File v8f = new File(new File(v1.getParent().toUri()), "v8");
+ Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
+ Path v8 = new Path(v8f.toURI());
+
+ File v2f = new File(v2.toUri());
+ File v9f = new File(new File(v2.getParent().toUri()), "v9");
+ Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
+ Path v9 = new Path(v9f.toURI());
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
+ conf.set(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(), v1 + " " + v8 + "," + v2 + " " + v9);
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volumes were replaced
+ cluster.start();
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+
+ // verify writes to new dir
+ getConnector().tableOperations().compact(tableNames[0], null, null, true, true);
+ getConnector().tableOperations().compact(tableNames[1], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+
+ // check that root tablet is not on volume 1 or 2
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers())) + RootTable.ZROOT_TABLET_PATH;
+ ZooKeeper zookeeper = new ZooKeeper(cluster.getZooKeepers(), 30000, null);
- String rootTabletDir = new String(zookeeper.getData(zpath, false, null), Constants.UTF8);
++ String rootTabletDir = new String(zookeeper.getData(zpath, false, null), UTF_8);
+ Assert.assertTrue(rootTabletDir.startsWith(v8.toString()) || rootTabletDir.startsWith(v9.toString()));
+ zookeeper.close();
+
+ getConnector().tableOperations().clone(tableNames[1], tableNames[2], true, new HashMap<String,String>(), new HashSet<String>());
+
+ getConnector().tableOperations().flush(MetadataTable.NAME, null, null, true);
+ getConnector().tableOperations().flush(RootTable.NAME, null, null, true);
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+ verifyVolumesUsed(tableNames[2], true, v8, v9);
+ }
+
+ @Test
+ public void testCleanReplaceVolumes() throws Exception {
+ testReplaceVolume(true);
+ }
+
+ @Test
+ public void testDirtyReplaceVolumes() throws Exception {
+ testReplaceVolume(false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
index f657034,0000000..05de342
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/AddSplitIT.java
@@@ -1,140 -1,0 +1,141 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class AddSplitIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void addSplitTest() throws Exception {
+
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+
+ insertData(tableName, 1l);
+
+ TreeSet<Text> splits = new TreeSet<Text>();
+ splits.add(new Text(String.format("%09d", 333)));
+ splits.add(new Text(String.format("%09d", 666)));
+
+ c.tableOperations().addSplits(tableName, splits);
+
+ UtilWaitThread.sleep(100);
+
+ Collection<Text> actualSplits = c.tableOperations().listSplits(tableName);
+
+ if (!splits.equals(new TreeSet<Text>(actualSplits))) {
+ throw new Exception(splits + " != " + actualSplits);
+ }
+
+ verifyData(tableName, 1l);
+ insertData(tableName, 2l);
+
+ // did not clear splits on purpose, it should ignore existing split points
+ // and still create the three additional split points
+
+ splits.add(new Text(String.format("%09d", 200)));
+ splits.add(new Text(String.format("%09d", 500)));
+ splits.add(new Text(String.format("%09d", 800)));
+
+ c.tableOperations().addSplits(tableName, splits);
+
+ UtilWaitThread.sleep(100);
+
+ actualSplits = c.tableOperations().listSplits(tableName);
+
+ if (!splits.equals(new TreeSet<Text>(actualSplits))) {
+ throw new Exception(splits + " != " + actualSplits);
+ }
+
+ verifyData(tableName, 2l);
+ }
+
+ private void verifyData(String tableName, long ts) throws Exception {
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ for (int i = 0; i < 10000; i++) {
+ if (!iter.hasNext()) {
+ throw new Exception("row " + i + " not found");
+ }
+
+ Entry<Key,Value> entry = iter.next();
+
+ String row = String.format("%09d", i);
+
+ if (!entry.getKey().getRow().equals(new Text(row))) {
+ throw new Exception("unexpected row " + entry.getKey() + " " + i);
+ }
+
+ if (entry.getKey().getTimestamp() != ts) {
+ throw new Exception("unexpected ts " + entry.getKey() + " " + ts);
+ }
+
+ if (Integer.parseInt(entry.getValue().toString()) != i) {
+ throw new Exception("unexpected value " + entry + " " + i);
+ }
+ }
+
+ if (iter.hasNext()) {
+ throw new Exception("found more than expected " + iter.next());
+ }
+
+ }
+
+ private void insertData(String tableName, long ts) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
+ BatchWriter bw = getConnector().createBatchWriter(tableName, null);
+
+ for (int i = 0; i < 10000; i++) {
+ String row = String.format("%09d", i);
+
+ Mutation m = new Mutation(new Text(row));
- m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), ts, new Value(Integer.toString(i).getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
index 0139a10,0000000..9c4492e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BadIteratorMincIT.java
@@@ -1,110 -1,0 +1,110 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.EnumSet;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BadIteratorMincIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ IteratorSetting is = new IteratorSetting(30, BadIterator.class);
+ c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+
+ Mutation m = new Mutation(new Text("r1"));
- m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++ m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ bw.close();
+
+ c.tableOperations().flush(tableName, null, null, false);
+ UtilWaitThread.sleep(1000);
+
+ // minc should fail, so there should be no files
+ FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 0, 0);
+
+ // try to scan table
+ Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+ int count = FunctionalTestUtils.count(scanner);
+ assertEquals("Did not see expected # entries " + count, 1, count);
+
+ // remove the bad iterator
+ c.tableOperations().removeIterator(tableName, BadIterator.class.getSimpleName(), EnumSet.of(IteratorScope.minc));
+
+ UtilWaitThread.sleep(5000);
+
+ // minc should complete
+ FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 1, 1);
+
+ count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 1)
+ throw new Exception("Did not see expected # entries " + count);
+
+ // now try putting bad iterator back and deleting the table
+ c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc));
+ bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ m = new Mutation(new Text("r2"));
- m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(Constants.UTF8)));
++ m.put(new Text("acf"), new Text(tableName), new Value("1".getBytes(UTF_8)));
+ bw.addMutation(m);
+ bw.close();
+
+ // make sure property is given time to propagate
+ UtilWaitThread.sleep(500);
+
+ c.tableOperations().flush(tableName, null, null, false);
+
+ // make sure the flush has time to start
+ UtilWaitThread.sleep(1000);
+
+ // this should not hang
+ c.tableOperations().delete(tableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
index c67407d,0000000..688a326
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchScanSplitIT.java
@@@ -1,125 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Random;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BatchScanSplitIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "0"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ int numRows = 1 << 18;
+
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 0; i < numRows; i++) {
+ Mutation m = new Mutation(new Text(String.format("%09x", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value(String.format("%016x", numRows - i).getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ getConnector().tableOperations().flush(tableName, null, null, true);
+
+ getConnector().tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "4K");
+
+ Collection<Text> splits = getConnector().tableOperations().listSplits(tableName);
+ while (splits.size() < 2) {
+ UtilWaitThread.sleep(1);
+ splits = getConnector().tableOperations().listSplits(tableName);
+ }
+
+ System.out.println("splits : " + splits);
+
+ Random random = new Random(19011230);
+ HashMap<Text,Value> expected = new HashMap<Text,Value>();
+ ArrayList<Range> ranges = new ArrayList<Range>();
+ for (int i = 0; i < 100; i++) {
+ int r = random.nextInt(numRows);
+ Text row = new Text(String.format("%09x", r));
- expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(Constants.UTF8)));
++ expected.put(row, new Value(String.format("%016x", numRows - r).getBytes(UTF_8)));
+ ranges.add(new Range(row));
+ }
+
+ // logger.setLevel(Level.TRACE);
+
+ HashMap<Text,Value> found = new HashMap<Text,Value>();
+
+ for (int i = 0; i < 20; i++) {
+ BatchScanner bs = getConnector().createBatchScanner(tableName, Authorizations.EMPTY, 4);
+
+ found.clear();
+
+ long t1 = System.currentTimeMillis();
+
+ bs.setRanges(ranges);
+
+ for (Entry<Key,Value> entry : bs) {
+ found.put(entry.getKey().getRow(), entry.getValue());
+ }
+ bs.close();
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("rate : %06.2f%n", ranges.size() / ((t2 - t1) / 1000.0)));
+
+ if (!found.equals(expected))
+ throw new Exception("Found and expected differ " + found + " " + expected);
+ }
+
+ splits = getConnector().tableOperations().listSplits(tableName);
+ log.info("splits : " + splits);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
index e2dd903,0000000..465936e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BatchWriterFlushIT.java
@@@ -1,181 -1,0 +1,182 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BatchWriterFlushIT extends SimpleMacIT {
+
+ private static final int NUM_TO_FLUSH = 100000;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 90;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = getUniqueNames(2);
+ String bwft = tableNames[0];
+ c.tableOperations().create(bwft);
+ String bwlt = tableNames[1];
+ c.tableOperations().create(bwlt);
+ runFlushTest(bwft);
+ runLatencyTest(bwlt);
+
+ }
+
+ private void runLatencyTest(String tableName) throws Exception {
+ // should automatically flush after 2 seconds
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(1000, TimeUnit.MILLISECONDS));
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+
+ Mutation m = new Mutation(new Text(String.format("r_%10d", 1)));
- m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(Constants.UTF8)));
++ m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(UTF_8)));
+ bw.addMutation(m);
+
+ UtilWaitThread.sleep(500);
+
+ int count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 0) {
+ throw new Exception("Flushed too soon");
+ }
+
+ UtilWaitThread.sleep(1500);
+
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 1) {
+ throw new Exception("Did not flush");
+ }
+
+ bw.close();
+ }
+
+ private void runFlushTest(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException,
+ Exception {
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ Random r = new Random();
+
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < NUM_TO_FLUSH; j++) {
+ int row = i * NUM_TO_FLUSH + j;
+
+ Mutation m = new Mutation(new Text(String.format("r_%10d", row)));
+ m.put(new Text("cf"), new Text("cq"), new Value(("" + row).getBytes()));
+ bw.addMutation(m);
+ }
+
+ bw.flush();
+
+ // do a few random lookups into the data just flushed
+
+ for (int k = 0; k < 10; k++) {
+ int rowToLookup = r.nextInt(NUM_TO_FLUSH) + i * NUM_TO_FLUSH;
+
+ scanner.setRange(new Range(new Text(String.format("r_%10d", rowToLookup))));
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ if (!iter.hasNext())
+ throw new Exception(" row " + rowToLookup + " not found after flush");
+
+ Entry<Key,Value> entry = iter.next();
+
+ if (iter.hasNext())
+ throw new Exception("Scanner returned too much");
+
+ verifyEntry(rowToLookup, entry);
+ }
+
+ // scan all data just flushed
+ scanner.setRange(new Range(new Text(String.format("r_%10d", i * NUM_TO_FLUSH)), true, new Text(String.format("r_%10d", (i + 1) * NUM_TO_FLUSH)), false));
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ for (int j = 0; j < NUM_TO_FLUSH; j++) {
+ int row = i * NUM_TO_FLUSH + j;
+
+ if (!iter.hasNext())
+ throw new Exception("Scan stopped permaturely at " + row);
+
+ Entry<Key,Value> entry = iter.next();
+
+ verifyEntry(row, entry);
+ }
+
+ if (iter.hasNext())
+ throw new Exception("Scanner returned too much");
+
+ }
+
+ bw.close();
+
+ // test adding a mutation to a closed batch writer
+ boolean caught = false;
+ try {
+ bw.addMutation(new Mutation(new Text("foobar")));
+ } catch (IllegalStateException ise) {
+ caught = true;
+ }
+
+ if (!caught) {
+ throw new Exception("Adding to closed batch writer did not fail");
+ }
+ }
+
+ private void verifyEntry(int row, Entry<Key,Value> entry) throws Exception {
+ if (!entry.getKey().getRow().toString().equals(String.format("r_%10d", row))) {
+ throw new Exception("Unexpected key returned, expected " + row + " got " + entry.getKey());
+ }
+
+ if (!entry.getValue().toString().equals("" + row)) {
+ throw new Exception("Unexpected value, expected " + row + " got " + entry.getValue());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 7974c6c,0000000..ffe55bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@@ -1,120 -1,0 +1,121 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.trace.TraceFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class BulkFileIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void testBulkFile() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (String split : "0333 0666 0999 1333 1666".split(" "))
+ splits.add(new Text(split));
+ c.tableOperations().addSplits(tableName, splits);
+ Configuration conf = new Configuration();
+ AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
+
+ String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
+
+ fs.delete(new Path(dir), true);
+
+ FileSKVWriter writer1 = FileOperations.getInstance().openWriter(dir + "/f1." + RFile.EXTENSION, fs, conf, aconf);
+ writer1.startDefaultLocalityGroup();
+ writeData(writer1, 0, 333);
+ writer1.close();
+
+ FileSKVWriter writer2 = FileOperations.getInstance().openWriter(dir + "/f2." + RFile.EXTENSION, fs, conf, aconf);
+ writer2.startDefaultLocalityGroup();
+ writeData(writer2, 334, 999);
+ writer2.close();
+
+ FileSKVWriter writer3 = FileOperations.getInstance().openWriter(dir + "/f3." + RFile.EXTENSION, fs, conf, aconf);
+ writer3.startDefaultLocalityGroup();
+ writeData(writer3, 1000, 1999);
+ writer3.close();
+
+ FunctionalTestUtils.bulkImport(c, fs, tableName, dir);
+
+ FunctionalTestUtils.checkRFiles(c, tableName, 6, 6, 1, 1);
+
+ verifyData(tableName, 0, 1999);
+
+ }
+
+ private void verifyData(String table, int s, int e) throws Exception {
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ for (int i = s; i <= e; i++) {
+ if (!iter.hasNext())
+ throw new Exception("row " + i + " not found");
+
+ Entry<Key,Value> entry = iter.next();
+
+ String row = String.format("%04d", i);
+
+ if (!entry.getKey().getRow().equals(new Text(row)))
+ throw new Exception("unexpected row " + entry.getKey() + " " + i);
+
+ if (Integer.parseInt(entry.getValue().toString()) != i)
+ throw new Exception("unexpected value " + entry + " " + i);
+ }
+
+ if (iter.hasNext())
+ throw new Exception("found more than expected " + iter.next());
+ }
+
+ private void writeData(FileSKVWriter w, int s, int e) throws Exception {
+ for (int i = s; i <= e; i++) {
- w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ w.append(new Key(new Text(String.format("%04d", i))), new Value(Integer.toString(i).getBytes(UTF_8)));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
index 785a997,0000000..92bd714
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java
@@@ -1,147 -1,0 +1,148 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collections;
+import java.util.EnumSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ConcurrencyIT extends ConfigurableMacIT {
+
+ static class ScanTask extends Thread {
+
+ int count = 0;
+ Scanner scanner;
+
+ ScanTask(Connector conn, long time) throws Exception {
+ scanner = conn.createScanner("cct", Authorizations.EMPTY);
+ IteratorSetting slow = new IteratorSetting(30, "slow", SlowIterator.class);
+ SlowIterator.setSleepTime(slow, time);
+ scanner.addScanIterator(slow);
+ }
+
+ @Override
+ public void run() {
+ count = FunctionalTestUtils.count(scanner);
+ }
+
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ /*
+ * Below is a diagram of the operations in this test over time.
+ *
+ * Scan 0 |------------------------------| Scan 1 |----------| Minc 1 |-----| Scan 2 |----------| Scan 3 |---------------| Minc 2 |-----| Majc 1 |-----|
+ */
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ runTest(c);
+ }
+
+ static void runTest(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
+ MutationsRejectedException, Exception, InterruptedException {
+ c.tableOperations().create("cct");
+ IteratorSetting is = new IteratorSetting(10, SlowIterator.class);
+ SlowIterator.setSleepTime(is, 50);
+ c.tableOperations().attachIterator("cct", is, EnumSet.of(IteratorScope.minc, IteratorScope.majc));
+ c.tableOperations().setProperty("cct", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
+
+ BatchWriter bw = c.createBatchWriter("cct", new BatchWriterConfig());
+ for (int i = 0; i < 50; i++) {
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+ bw.flush();
+
+ ScanTask st0 = new ScanTask(c, 300);
+ st0.start();
+
+ ScanTask st1 = new ScanTask(c, 100);
+ st1.start();
+
+ UtilWaitThread.sleep(50);
+ c.tableOperations().flush("cct", null, null, true);
+
+ for (int i = 0; i < 50; i++) {
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
- m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++ m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+
+ bw.flush();
+
+ ScanTask st2 = new ScanTask(c, 100);
+ st2.start();
+
+ st1.join();
+ st2.join();
+ if (st1.count != 50)
+ throw new Exception("Thread 1 did not see 50, saw " + st1.count);
+
+ if (st2.count != 50)
+ throw new Exception("Thread 2 did not see 50, saw " + st2.count);
+
+ ScanTask st3 = new ScanTask(c, 150);
+ st3.start();
+
+ UtilWaitThread.sleep(50);
+ c.tableOperations().flush("cct", null, null, false);
+
+ st3.join();
+ if (st3.count != 50)
+ throw new Exception("Thread 3 did not see 50, saw " + st3.count);
+
+ st0.join();
+ if (st0.count != 50)
+ throw new Exception("Thread 0 did not see 50, saw " + st0.count);
+
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
index d5e9188,0000000..7e5944e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConstraintIT.java
@@@ -1,317 -1,0 +1,318 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ConstraintIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 30;
+ }
+
+ @Test
+ public void run() throws Exception {
+ String[] tableNames = getUniqueNames(3);
+ Connector c = getConnector();
+ for (String table : tableNames) {
+ c.tableOperations().create(table);
+ c.tableOperations().addConstraint(table, NumericValueConstraint.class.getName());
+ c.tableOperations().addConstraint(table, AlphaNumKeyConstraint.class.getName());
+ }
+
+ // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
+ // logger.setLevel(Level.TRACE);
+
+ test1(tableNames[0]);
+
+ // logger.setLevel(Level.TRACE);
+
+ test2(tableNames[1], false);
+ test2(tableNames[2], true);
+ }
+
+ private void test1(String tableName) throws Exception {
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ Mutation mut1 = new Mutation(new Text("r1"));
- mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(Constants.UTF8)));
++ mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(UTF_8)));
+
+ bw.addMutation(mut1);
+
+ // should not throw any exceptions
+ bw.close();
+
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ // create a mutation with a non numeric value
+ Mutation mut2 = new Mutation(new Text("r1"));
- mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(Constants.UTF8)));
++ mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(UTF_8)));
+
+ bw.addMutation(mut2);
+
+ boolean sawMRE = false;
+
+ try {
+ bw.close();
+ // should not get here
+ throw new Exception("Test failed, constraint did not catch bad mutation");
+ } catch (MutationsRejectedException mre) {
+ sawMRE = true;
+
+ // verify constraint violation summary
+ List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
+
+ if (cvsl.size() != 1) {
+ throw new Exception("Unexpected constraints");
+ }
+
+ for (ConstraintViolationSummary cvs : cvsl) {
+ if (!cvs.constrainClass.equals(NumericValueConstraint.class.getName())) {
+ throw new Exception("Unexpected constraint class " + cvs.constrainClass);
+ }
+
+ if (cvs.numberOfViolatingMutations != 1) {
+ throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
+ }
+ }
+ }
+
+ if (!sawMRE) {
+ throw new Exception("Did not see MutationsRejectedException");
+ }
+
+ // verify mutation did not go through
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ scanner.setRange(new Range(new Text("r1")));
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ Entry<Key,Value> entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ // remove the numeric value constraint
+ getConnector().tableOperations().removeConstraint(tableName, 2);
+ UtilWaitThread.sleep(1000);
+
+ // now should be able to add a non numeric value
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+ bw.addMutation(mut2);
+ bw.close();
+
+ // verify mutation went through
+ iter = scanner.iterator();
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ // add a constraint that references a non-existant class
+ getConnector().tableOperations().setProperty(tableName, Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistantClass");
+ UtilWaitThread.sleep(1000);
+
+ // add a mutation
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ Mutation mut3 = new Mutation(new Text("r1"));
- mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
++ mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
+
+ bw.addMutation(mut3);
+
+ sawMRE = false;
+
+ try {
+ bw.close();
+ // should not get here
+ throw new Exception("Test failed, mutation went through when table had bad constraints");
+ } catch (MutationsRejectedException mre) {
+ sawMRE = true;
+ }
+
+ if (!sawMRE) {
+ throw new Exception("Did not see MutationsRejectedException");
+ }
+
+ // verify the mutation did not go through
+ iter = scanner.iterator();
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ // remove the bad constraint
+ getConnector().tableOperations().removeConstraint(tableName, 1);
+ UtilWaitThread.sleep(1000);
+
+ // try the mutation again
+ bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+ bw.addMutation(mut3);
+ bw.close();
+
+ // verify it went through
+ iter = scanner.iterator();
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+ }
+
+ private Mutation newMut(String row, String cf, String cq, String val) {
+ Mutation mut1 = new Mutation(new Text(row));
- mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(Constants.UTF8)));
++ mut1.put(new Text(cf), new Text(cq), new Value(val.getBytes(UTF_8)));
+ return mut1;
+ }
+
+ private void test2(String table, boolean doFlush) throws Exception {
+ // test sending multiple mutations with multiple constrain violations... all of the non violating mutations
+ // should go through
+ int numericErrors = 2;
+
+ BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ bw.addMutation(newMut("r1", "cf1", "cq1", "123"));
+ bw.addMutation(newMut("r1", "cf1", "cq2", "I'm a bad value"));
+ if (doFlush) {
+ try {
+ bw.flush();
+ throw new Exception("Didn't find a bad mutation");
+ } catch (MutationsRejectedException mre) {
+ // ignored
+ try {
+ bw.close();
+ } catch (MutationsRejectedException ex) {
+ // ignored
+ }
+ bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ numericErrors = 1;
+ }
+ }
+ bw.addMutation(newMut("r1", "cf1", "cq3", "I'm a naughty value"));
+ bw.addMutation(newMut("@bad row@", "cf1", "cq2", "456"));
+ bw.addMutation(newMut("r1", "cf1", "cq4", "789"));
+
+ boolean sawMRE = false;
+
+ try {
+ bw.close();
+ // should not get here
+ throw new Exception("Test failed, constraint did not catch bad mutation");
+ } catch (MutationsRejectedException mre) {
+ System.out.println(mre);
+
+ sawMRE = true;
+
+ // verify constraint violation summary
+ List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
+
+ if (cvsl.size() != 2) {
+ throw new Exception("Unexpected constraints");
+ }
+
+ HashMap<String,Integer> expected = new HashMap<String,Integer>();
+
+ expected.put("org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", numericErrors);
+ expected.put("org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint", 1);
+
+ for (ConstraintViolationSummary cvs : cvsl) {
+ if (expected.get(cvs.constrainClass) != cvs.numberOfViolatingMutations) {
+ throw new Exception("Unexpected " + cvs.constrainClass + " " + cvs.numberOfViolatingMutations);
+ }
+ }
+ }
+
+ if (!sawMRE) {
+ throw new Exception("Did not see MutationsRejectedException");
+ }
+
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+
+ Entry<Key,Value> entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ entry = iter.next();
+
+ if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
- || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(Constants.UTF8)))) {
++ || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(UTF_8)))) {
+ throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ if (iter.hasNext()) {
+ entry = iter.next();
+ throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
index b9b54c7,0000000..5b5249b
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateAndUseIT.java
@@@ -1,128 -1,0 +1,129 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class CreateAndUseIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private static SortedSet<Text> splits;
+
+ @BeforeClass
+ public static void createData() throws Exception {
+ splits = new TreeSet<Text>();
+
+ for (int i = 1; i < 256; i++) {
+ splits.add(new Text(String.format("%08x", i << 8)));
+ }
+ }
+
+ @Test
+ public void verifyDataIsPresent() throws Exception {
+ Text cf = new Text("cf1");
+ Text cq = new Text("cq1");
+
+ String tableName = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(tableName);
+ getConnector().tableOperations().addSplits(tableName, splits);
+ BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 1; i < 257; i++) {
+ Mutation m = new Mutation(new Text(String.format("%08x", (i << 8) - 16)));
- m.put(cf, cq, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
++ m.put(cf, cq, new Value(Integer.toString(i).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ Scanner scanner1 = getConnector().createScanner(tableName, Authorizations.EMPTY);
+
+ int ei = 1;
+
+ for (Entry<Key,Value> entry : scanner1) {
+ Assert.assertEquals(String.format("%08x", (ei << 8) - 16), entry.getKey().getRow().toString());
+ Assert.assertEquals(Integer.toString(ei), entry.getValue().toString());
+
+ ei++;
+ }
+
+ Assert.assertEquals("Did not see expected number of rows", 257, ei);
+ }
+
+ @Test
+ public void createTableAndScan() throws Exception {
+ String table2 = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(table2);
+ getConnector().tableOperations().addSplits(table2, splits);
+ Scanner scanner2 = getConnector().createScanner(table2, Authorizations.EMPTY);
+ int count = 0;
+ for (Entry<Key,Value> entry : scanner2) {
+ if (entry != null)
+ count++;
+ }
+
+ if (count != 0) {
+ throw new Exception("Did not see expected number of entries, count = " + count);
+ }
+ }
+
+ @Test
+ public void createTableAndBatchScan() throws Exception {
+ ArrayList<Range> ranges = new ArrayList<Range>();
+ for (int i = 1; i < 257; i++) {
+ ranges.add(new Range(new Text(String.format("%08x", (i << 8) - 16))));
+ }
+
+ String table3 = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(table3);
+ getConnector().tableOperations().addSplits(table3, splits);
+ BatchScanner bs = getConnector().createBatchScanner(table3, Authorizations.EMPTY, 3);
+ bs.setRanges(ranges);
+ Iterator<Entry<Key,Value>> iter = bs.iterator();
+ int count = Iterators.size(iter);
+ bs.close();
+
+ Assert.assertEquals("Did not expect to find any entries", 0, count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
index 83b5b24,0000000..0578ef4
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteEverythingIT.java
@@@ -1,94 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.*;
+
+import java.util.Collections;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class DeleteEverythingIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "1s"));
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("de");
+ BatchWriter bw = getConnector().createBatchWriter("de", new BatchWriterConfig());
+ Mutation m = new Mutation(new Text("foo"));
- m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(Constants.UTF8)));
++ m.put(new Text("bar"), new Text("1910"), new Value("5".getBytes(UTF_8)));
+ bw.addMutation(m);
+ bw.flush();
+
+ getConnector().tableOperations().flush("de", null, null, true);
+
+ FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 1, 1);
+
+ m = new Mutation(new Text("foo"));
+ m.putDelete(new Text("bar"), new Text("1910"));
+ bw.addMutation(m);
+ bw.flush();
+
+ Scanner scanner = getConnector().createScanner("de", Authorizations.EMPTY);
+ scanner.setRange(new Range());
+ int count = FunctionalTestUtils.count(scanner);
+ assertEquals("count == " + count, 0, count);
+ getConnector().tableOperations().flush("de", null, null, true);
+
+ getConnector().tableOperations().setProperty("de", Property.TABLE_MAJC_RATIO.getKey(), "1.0");
+ UtilWaitThread.sleep(4000);
+
+ FunctionalTestUtils.checkRFiles(c, "de", 1, 1, 0, 0);
+
+ bw.close();
+
+ count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> entry : scanner) {
+ count++;
+ }
+
+ if (count != 0)
+ throw new Exception("count == " + count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
index a8237a6,0000000..d35ba9f
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java
@@@ -1,143 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+
+// attempt to reproduce ACCUMULO-315
+public class DeleteRowsSplitIT extends SimpleMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private static final Logger log = Logger.getLogger(DeleteRowsSplitIT.class);
+
+ private static final String LETTERS = "abcdefghijklmnopqrstuvwxyz";
+ static final SortedSet<Text> SPLITS = new TreeSet<Text>();
+ static final List<String> ROWS = new ArrayList<String>();
+ static {
- for (byte b : LETTERS.getBytes(Constants.UTF8)) {
++ for (byte b : LETTERS.getBytes(UTF_8)) {
+ SPLITS.add(new Text(new byte[] {b}));
- ROWS.add(new String(new byte[] {b}, Constants.UTF8));
++ ROWS.add(new String(new byte[] {b}, UTF_8));
+ }
+ }
+
+ @Test
+ public void run() throws Exception {
+ // Delete ranges of rows, and verify the are removed
+ // Do this while adding many splits
+ final String tableName = getUniqueNames(1)[0];
+
+ // Eliminate whole tablets
+ for (int test = 0; test < 10; test++) {
+ // create a table
+ log.info("Test " + test);
+ getConnector().tableOperations().create(tableName);
+
+ // put some data in it
+ fillTable(tableName);
+
+ // generate a random delete range
+ final Text start = new Text();
+ final Text end = new Text();
+ generateRandomRange(start, end);
+
+ // initiate the delete range
+ final boolean fail[] = {false};
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ // split the table
+ final SortedSet<Text> afterEnd = SPLITS.tailSet(new Text(end.toString() + "\0"));
+ getConnector().tableOperations().addSplits(tableName, afterEnd);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ synchronized (fail) {
+ fail[0] = true;
+ }
+ }
+ }
+ };
+ t.start();
+
+ UtilWaitThread.sleep(test * 2);
+
+ getConnector().tableOperations().deleteRows(tableName, start, end);
+
+ t.join();
+ synchronized (fail) {
+ assertTrue(!fail[0]);
+ }
+
+ // scan the table
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ assertTrue(row.compareTo(start) <= 0 || row.compareTo(end) > 0);
+ }
+
+ // delete the table
+ getConnector().tableOperations().delete(tableName);
+ }
+ }
+
+ private void generateRandomRange(Text start, Text end) {
+ List<String> bunch = new ArrayList<String>(ROWS);
+ Collections.shuffle(bunch);
+ if (bunch.get(0).compareTo((bunch.get(1))) < 0) {
+ start.set(bunch.get(0));
+ end.set(bunch.get(1));
+ } else {
+ start.set(bunch.get(1));
+ end.set(bunch.get(0));
+ }
+
+ }
+
+ private void fillTable(String table) throws Exception {
+ BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ for (String row : ROWS) {
+ Mutation m = new Mutation(row);
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+}
[27/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index 0778f5b,0000000..3459069
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@@ -1,3857 -1,0 +1,3859 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSource;
+import org.apache.accumulo.core.iterators.system.StatsIterator;
+import org.apache.accumulo.core.iterators.system.VisibilityFilter;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.fs.VolumeUtil.TabletFiles;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.tableOps.CompactionIterators;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.tablets.TabletTime;
+import org.apache.accumulo.server.tablets.UniqueNameAllocator;
+import org.apache.accumulo.server.util.FileUtil;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.TabletOperations;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.tserver.Compactor.CompactionCanceledException;
+import org.apache.accumulo.tserver.Compactor.CompactionEnv;
+import org.apache.accumulo.tserver.FileManager.ScanFileManager;
+import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
+import org.apache.accumulo.tserver.TabletServer.TservConstraintEnv;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.apache.accumulo.tserver.compaction.WriteParameters;
+import org.apache.accumulo.tserver.constraints.ConstraintChecker;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/*
+ * We need to be able to have the master tell a tabletServer to
+ * close this file, and the tablet server to handle all pending client reads
+ * before closing
+ *
+ */
+
+/**
+ *
+ * this class just provides an interface to read from a MapFile mostly takes care of reporting start and end keys
+ *
+ * need this because a single row extent can have multiple columns this manages all the columns (each handled by a store) for a single row-extent
+ *
+ *
+ */
+
+public class Tablet {
+
+ enum MinorCompactionReason {
+ USER, SYSTEM, CLOSE, RECOVERY
+ }
+
+ public class CommitSession {
+
+ private int seq;
+ private InMemoryMap memTable;
+ private int commitsInProgress;
+ private long maxCommittedTime = Long.MIN_VALUE;
+
+ private CommitSession(int seq, InMemoryMap imm) {
+ this.seq = seq;
+ this.memTable = imm;
+ commitsInProgress = 0;
+ }
+
+ public int getWALogSeq() {
+ return seq;
+ }
+
+ private void decrementCommitsInProgress() {
+ if (commitsInProgress < 1)
+ throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
+
+ commitsInProgress--;
+ if (commitsInProgress == 0)
+ Tablet.this.notifyAll();
+ }
+
+ private void incrementCommitsInProgress() {
+ if (commitsInProgress < 0)
+ throw new IllegalStateException("commitsInProgress = " + commitsInProgress);
+
+ commitsInProgress++;
+ }
+
+ private void waitForCommitsToFinish() {
+ while (commitsInProgress > 0) {
+ try {
+ Tablet.this.wait(50);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ public void abortCommit(List<Mutation> value) {
+ Tablet.this.abortCommit(this, value);
+ }
+
+ public void commit(List<Mutation> mutations) {
+ Tablet.this.commit(this, mutations);
+ }
+
+ public Tablet getTablet() {
+ return Tablet.this;
+ }
+
+ public boolean beginUpdatingLogsUsed(ArrayList<DfsLogger> copy, boolean mincFinish) {
+ return Tablet.this.beginUpdatingLogsUsed(memTable, copy, mincFinish);
+ }
+
+ public void finishUpdatingLogsUsed() {
+ Tablet.this.finishUpdatingLogsUsed();
+ }
+
+ public int getLogId() {
+ return logId;
+ }
+
+ public KeyExtent getExtent() {
+ return extent;
+ }
+
+ private void updateMaxCommittedTime(long time) {
+ maxCommittedTime = Math.max(time, maxCommittedTime);
+ }
+
+ private long getMaxCommittedTime() {
+ if (maxCommittedTime == Long.MIN_VALUE)
+ throw new IllegalStateException("Tried to read max committed time when it was never set");
+ return maxCommittedTime;
+ }
+
+ }
+
+ private class TabletMemory {
+ private InMemoryMap memTable;
+ private InMemoryMap otherMemTable;
+ private InMemoryMap deletingMemTable;
+ private int nextSeq = 1;
+ private CommitSession commitSession;
+
+ TabletMemory() {
+ try {
+ memTable = new InMemoryMap(acuTableConf);
+ } catch (LocalityGroupConfigurationError e) {
+ throw new RuntimeException(e);
+ }
+ commitSession = new CommitSession(nextSeq, memTable);
+ nextSeq += 2;
+ }
+
+ InMemoryMap getMemTable() {
+ return memTable;
+ }
+
+ InMemoryMap getMinCMemTable() {
+ return otherMemTable;
+ }
+
+ CommitSession prepareForMinC() {
+ if (otherMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ if (deletingMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ otherMemTable = memTable;
+ try {
+ memTable = new InMemoryMap(acuTableConf);
+ } catch (LocalityGroupConfigurationError e) {
+ throw new RuntimeException(e);
+ }
+
+ CommitSession oldCommitSession = commitSession;
+ commitSession = new CommitSession(nextSeq, memTable);
+ nextSeq += 2;
+
+ tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), otherMemTable.estimatedSizeInBytes());
+
+ return oldCommitSession;
+ }
+
+ void finishedMinC() {
+
+ if (otherMemTable == null) {
+ throw new IllegalStateException();
+ }
+
+ if (deletingMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ deletingMemTable = otherMemTable;
+
+ otherMemTable = null;
+ Tablet.this.notifyAll();
+ }
+
+ void finalizeMinC() {
+ try {
+ deletingMemTable.delete(15000);
+ } finally {
+ synchronized (Tablet.this) {
+ if (otherMemTable != null) {
+ throw new IllegalStateException();
+ }
+
+ if (deletingMemTable == null) {
+ throw new IllegalStateException();
+ }
+
+ deletingMemTable = null;
+
+ tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), 0);
+ }
+ }
+ }
+
+ boolean memoryReservedForMinC() {
+ return otherMemTable != null || deletingMemTable != null;
+ }
+
+ void waitForMinC() {
+ while (otherMemTable != null || deletingMemTable != null) {
+ try {
+ Tablet.this.wait(50);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ void mutate(CommitSession cm, List<Mutation> mutations) {
+ cm.memTable.mutate(mutations);
+ }
+
+ void updateMemoryUsageStats() {
+ long other = 0;
+ if (otherMemTable != null)
+ other = otherMemTable.estimatedSizeInBytes();
+ else if (deletingMemTable != null)
+ other = deletingMemTable.estimatedSizeInBytes();
+
+ tabletResources.updateMemoryUsageStats(memTable.estimatedSizeInBytes(), other);
+ }
+
+ List<MemoryIterator> getIterators() {
+ List<MemoryIterator> toReturn = new ArrayList<MemoryIterator>(2);
+ toReturn.add(memTable.skvIterator());
+ if (otherMemTable != null)
+ toReturn.add(otherMemTable.skvIterator());
+ return toReturn;
+ }
+
+ void returnIterators(List<MemoryIterator> iters) {
+ for (MemoryIterator iter : iters) {
+ iter.close();
+ }
+ }
+
+ public long getNumEntries() {
+ if (otherMemTable != null)
+ return memTable.getNumEntries() + otherMemTable.getNumEntries();
+ return memTable.getNumEntries();
+ }
+
+ CommitSession getCommitSession() {
+ return commitSession;
+ }
+ }
+
+ private TabletMemory tabletMemory;
+
+ private final TabletTime tabletTime;
+ private long persistedTime;
+ private final Object timeLock = new Object();
+
+ private final Path location; // absolute path of this tablets dir
+ private TServerInstance lastLocation;
+
+ private Configuration conf;
+ private VolumeManager fs;
+
+ private TableConfiguration acuTableConf;
+
+ private volatile boolean tableDirChecked = false;
+
+ private AtomicLong dataSourceDeletions = new AtomicLong(0);
+ private Set<ScanDataSource> activeScans = new HashSet<ScanDataSource>();
+
+ private volatile boolean closing = false;
+ private boolean closed = false;
+ private boolean closeComplete = false;
+
+ private long lastFlushID = -1;
+ private long lastCompactID = -1;
+
+ private static class CompactionWaitInfo {
+ long flushID = -1;
+ long compactionID = -1;
+ }
+
+ // stores info about user initiated major compaction that is waiting on a minor compaction to finish
+ private CompactionWaitInfo compactionWaitInfo = new CompactionWaitInfo();
+
+ private KeyExtent extent;
+
+ private TabletResourceManager tabletResources;
+ final private DatafileManager datafileManager;
+ private volatile boolean majorCompactionInProgress = false;
+ private volatile boolean majorCompactionWaitingToStart = false;
+ private Set<MajorCompactionReason> majorCompactionQueued = Collections.synchronizedSet(EnumSet.noneOf(MajorCompactionReason.class));
+ private volatile boolean minorCompactionInProgress = false;
+ private volatile boolean minorCompactionWaitingToStart = false;
+
+ private boolean updatingFlushID = false;
+
+ private AtomicReference<ConstraintChecker> constraintChecker = new AtomicReference<ConstraintChecker>();
+
+ private final String tabletDirectory;
+
+ private int writesInProgress = 0;
+
+ private static final Logger log = Logger.getLogger(Tablet.class);
+ public TabletStatsKeeper timer;
+
+ private Rate queryRate = new Rate(0.2);
+ private long queryCount = 0;
+
+ private Rate queryByteRate = new Rate(0.2);
+ private long queryBytes = 0;
+
+ private Rate ingestRate = new Rate(0.2);
+ private long ingestCount = 0;
+
+ private Rate ingestByteRate = new Rate(0.2);
+ private long ingestBytes = 0;
+
+ private byte[] defaultSecurityLabel = new byte[0];
+
+ private long lastMinorCompactionFinishTime;
+ private long lastMapFileImportTime;
+
+ private volatile long numEntries;
+ private volatile long numEntriesInMemory;
+
+ // a count of the amount of data read by the iterators
+ private AtomicLong scannedCount = new AtomicLong(0);
+ private Rate scannedRate = new Rate(0.2);
+
+ private ConfigurationObserver configObserver;
+
+ private TabletServer tabletServer;
+
+ private final int logId;
+ // ensure we only have one reader/writer of our bulk file notes at at time
+ public final Object bulkFileImportLock = new Object();
+
+ public int getLogId() {
+ return logId;
+ }
+
+ public static class TabletClosedException extends RuntimeException {
+ public TabletClosedException(Exception e) {
+ super(e);
+ }
+
+ public TabletClosedException() {
+ super();
+ }
+
+ private static final long serialVersionUID = 1L;
+ }
+
+ FileRef getNextMapFilename(String prefix) throws IOException {
+ String extension = FileOperations.getNewFileExtension(tabletServer.getTableConfiguration(extent));
+ checkTabletDir();
+ return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension);
+ }
+
+ private void checkTabletDir() throws IOException {
+ if (!tableDirChecked) {
+ checkTabletDir(this.location);
+ tableDirChecked = true;
+ }
+ }
+
+ private void checkTabletDir(Path tabletDir) throws IOException {
+
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(tabletDir);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+
+ if (files == null) {
+ if (tabletDir.getName().startsWith(Constants.CLONE_PREFIX))
+ log.debug("Tablet " + extent + " had no dir, creating " + tabletDir); // its a clone dir...
+ else
+ log.warn("Tablet " + extent + " had no dir, creating " + tabletDir);
+
+ fs.mkdirs(tabletDir);
+ }
+ }
+
+ class DatafileManager {
+ // access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
+ final private Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new TreeMap<FileRef,DataFileValue>());
+
+ DatafileManager(SortedMap<FileRef,DataFileValue> datafileSizes) {
+ for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet())
+ this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
+ }
+
+ FileRef mergingMinorCompactionFile = null;
+ Set<FileRef> filesToDeleteAfterScan = new HashSet<FileRef>();
+ Map<Long,Set<FileRef>> scanFileReservations = new HashMap<Long,Set<FileRef>>();
+ MapCounter<FileRef> fileScanReferenceCounts = new MapCounter<FileRef>();
+ long nextScanReservationId = 0;
+ boolean reservationsBlocked = false;
+
+ Set<FileRef> majorCompactingFiles = new HashSet<FileRef>();
+
+ Pair<Long,Map<FileRef,DataFileValue>> reserveFilesForScan() {
+ synchronized (Tablet.this) {
+
+ while (reservationsBlocked) {
+ try {
+ Tablet.this.wait(50);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+
+ Set<FileRef> absFilePaths = new HashSet<FileRef>(datafileSizes.keySet());
+
+ long rid = nextScanReservationId++;
+
+ scanFileReservations.put(rid, absFilePaths);
+
+ Map<FileRef,DataFileValue> ret = new HashMap<FileRef,DataFileValue>();
+
+ for (FileRef path : absFilePaths) {
+ fileScanReferenceCounts.increment(path, 1);
+ ret.put(path, datafileSizes.get(path));
+ }
+
+ return new Pair<Long,Map<FileRef,DataFileValue>>(rid, ret);
+ }
+ }
+
+ void returnFilesForScan(Long reservationId) {
+
+ final Set<FileRef> filesToDelete = new HashSet<FileRef>();
+
+ synchronized (Tablet.this) {
+ Set<FileRef> absFilePaths = scanFileReservations.remove(reservationId);
+
+ if (absFilePaths == null)
+ throw new IllegalArgumentException("Unknown scan reservation id " + reservationId);
+
+ boolean notify = false;
+ for (FileRef path : absFilePaths) {
+ long refCount = fileScanReferenceCounts.decrement(path, 1);
+ if (refCount == 0) {
+ if (filesToDeleteAfterScan.remove(path))
+ filesToDelete.add(path);
+ notify = true;
+ } else if (refCount < 0)
+ throw new IllegalStateException("Scan ref count for " + path + " is " + refCount);
+ }
+
+ if (notify)
+ Tablet.this.notifyAll();
+ }
+
+ if (filesToDelete.size() > 0) {
+ log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
+ MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
+ }
+ }
+
+ private void removeFilesAfterScan(Set<FileRef> scanFiles) {
+ if (scanFiles.size() == 0)
+ return;
+
+ Set<FileRef> filesToDelete = new HashSet<FileRef>();
+
+ synchronized (Tablet.this) {
+ for (FileRef path : scanFiles) {
+ if (fileScanReferenceCounts.get(path) == 0)
+ filesToDelete.add(path);
+ else
+ filesToDeleteAfterScan.add(path);
+ }
+ }
+
+ if (filesToDelete.size() > 0) {
+ log.debug("Removing scan refs from metadata " + extent + " " + filesToDelete);
+ MetadataTableUtil.removeScanFiles(extent, filesToDelete, SystemCredentials.get(), tabletServer.getLock());
+ }
+ }
+
+ private TreeSet<FileRef> waitForScansToFinish(Set<FileRef> pathsToWaitFor, boolean blockNewScans, long maxWaitTime) {
+ long startTime = System.currentTimeMillis();
+ TreeSet<FileRef> inUse = new TreeSet<FileRef>();
+
+ Span waitForScans = Trace.start("waitForScans");
+ try {
+ synchronized (Tablet.this) {
+ if (blockNewScans) {
+ if (reservationsBlocked)
+ throw new IllegalStateException();
+
+ reservationsBlocked = true;
+ }
+
+ for (FileRef path : pathsToWaitFor) {
+ while (fileScanReferenceCounts.get(path) > 0 && System.currentTimeMillis() - startTime < maxWaitTime) {
+ try {
+ Tablet.this.wait(100);
+ } catch (InterruptedException e) {
+ log.warn(e, e);
+ }
+ }
+ }
+
+ for (FileRef path : pathsToWaitFor) {
+ if (fileScanReferenceCounts.get(path) > 0)
+ inUse.add(path);
+ }
+
+ if (blockNewScans) {
+ reservationsBlocked = false;
+ Tablet.this.notifyAll();
+ }
+
+ }
+ } finally {
+ waitForScans.stop();
+ }
+ return inUse;
+ }
+
+ public void importMapFiles(long tid, Map<FileRef,DataFileValue> pathsString, boolean setTime) throws IOException {
+
+ String bulkDir = null;
+
+ Map<FileRef,DataFileValue> paths = new HashMap<FileRef,DataFileValue>();
+ for (Entry<FileRef,DataFileValue> entry : pathsString.entrySet())
+ paths.put(entry.getKey(), entry.getValue());
+
+ for (FileRef tpath : paths.keySet()) {
+
+ boolean inTheRightDirectory = false;
+ Path parent = tpath.path().getParent().getParent();
+ for (String tablesDir : ServerConstants.getTablesDirs()) {
+ if (parent.equals(new Path(tablesDir, extent.getTableId().toString()))) {
+ inTheRightDirectory = true;
+ break;
+ }
+ }
+ if (!inTheRightDirectory) {
+ throw new IOException("Data file " + tpath + " not in table dirs");
+ }
+
+ if (bulkDir == null)
+ bulkDir = tpath.path().getParent().toString();
+ else if (!bulkDir.equals(tpath.path().getParent().toString()))
+ throw new IllegalArgumentException("bulk files in different dirs " + bulkDir + " " + tpath);
+
+ }
+
+ if (extent.isRootTablet()) {
+ throw new IllegalArgumentException("Can not import files to root tablet");
+ }
+
+ synchronized (bulkFileImportLock) {
+ Credentials creds = SystemCredentials.get();
+ Connector conn;
+ try {
+ conn = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ // Remove any bulk files we've previously loaded and compacted away
+ List<FileRef> files = MetadataTableUtil.getBulkFilesLoaded(conn, extent, tid);
+
+ for (FileRef file : files)
+ if (paths.keySet().remove(file))
+ log.debug("Ignoring request to re-import a file already imported: " + extent + ": " + file);
+
+ if (paths.size() > 0) {
+ long bulkTime = Long.MIN_VALUE;
+ if (setTime) {
+ for (DataFileValue dfv : paths.values()) {
+ long nextTime = tabletTime.getAndUpdateTime();
+ if (nextTime < bulkTime)
+ throw new IllegalStateException("Time went backwards unexpectedly " + nextTime + " " + bulkTime);
+ bulkTime = nextTime;
+ dfv.setTime(bulkTime);
+ }
+ }
+
+ synchronized (timeLock) {
+ if (bulkTime > persistedTime)
+ persistedTime = bulkTime;
+
+ MetadataTableUtil.updateTabletDataFile(tid, extent, paths, tabletTime.getMetadataValue(persistedTime), creds, tabletServer.getLock());
+ }
+ }
+ }
+
+ synchronized (Tablet.this) {
+ for (Entry<FileRef,DataFileValue> tpath : paths.entrySet()) {
+ if (datafileSizes.containsKey(tpath.getKey())) {
+ log.error("Adding file that is already in set " + tpath.getKey());
+ }
+ datafileSizes.put(tpath.getKey(), tpath.getValue());
+
+ }
+
+ tabletResources.importedMapFiles();
+
+ computeNumEntries();
+ }
+
+ for (Entry<FileRef,DataFileValue> entry : paths.entrySet()) {
+ log.log(TLevel.TABLET_HIST, extent + " import " + entry.getKey() + " " + entry.getValue());
+ }
+ }
+
+ FileRef reserveMergingMinorCompactionFile() {
+ if (mergingMinorCompactionFile != null)
+ throw new IllegalStateException("Tried to reserve merging minor compaction file when already reserved : " + mergingMinorCompactionFile);
+
+ if (extent.isRootTablet())
+ return null;
+
+ int maxFiles = acuTableConf.getMaxFilesPerTablet();
+
+ // when a major compaction is running and we are at max files, write out
+ // one extra file... want to avoid the case where major compaction is
+ // compacting everything except for the largest file, and therefore the
+ // largest file is returned for merging.. the following check mostly
+ // avoids this case, except for the case where major compactions fail or
+ // are canceled
+ if (majorCompactingFiles.size() > 0 && datafileSizes.size() == maxFiles)
+ return null;
+
+ if (datafileSizes.size() >= maxFiles) {
+ // find the smallest file
+
+ long min = Long.MAX_VALUE;
+ FileRef minName = null;
+
+ for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
+ if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey())) {
+ min = entry.getValue().getSize();
+ minName = entry.getKey();
+ }
+ }
+
+ if (minName == null)
+ return null;
+
+ mergingMinorCompactionFile = minName;
+ return minName;
+ }
+
+ return null;
+ }
+
+ void unreserveMergingMinorCompactionFile(FileRef file) {
+ if ((file == null && mergingMinorCompactionFile != null) || (file != null && mergingMinorCompactionFile == null)
+ || (file != null && mergingMinorCompactionFile != null && !file.equals(mergingMinorCompactionFile)))
+ throw new IllegalStateException("Disagreement " + file + " " + mergingMinorCompactionFile);
+
+ mergingMinorCompactionFile = null;
+ }
+
+ void bringMinorCompactionOnline(FileRef tmpDatafile, FileRef newDatafile, FileRef absMergeFile, DataFileValue dfv, CommitSession commitSession, long flushId)
+ throws IOException {
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (extent.isRootTablet()) {
+ try {
+ if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
+ throw new IllegalStateException();
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
+ }
+ }
+
+ // rename before putting in metadata table, so files in metadata table should
+ // always exist
+ do {
+ try {
+ if (dfv.getNumEntries() == 0) {
+ fs.deleteRecursively(tmpDatafile.path());
+ } else {
+ if (fs.exists(newDatafile.path())) {
+ log.warn("Target map file already exist " + newDatafile);
+ fs.deleteRecursively(newDatafile.path());
+ }
+
+ rename(fs, tmpDatafile.path(), newDatafile.path());
+ }
+ break;
+ } catch (IOException ioe) {
+ log.warn("Tablet " + extent + " failed to rename " + newDatafile + " after MinC, will retry in 60 secs...", ioe);
+ UtilWaitThread.sleep(60 * 1000);
+ }
+ } while (true);
+
+ long t1, t2;
+
+ // the code below always assumes merged files are in use by scans... this must be done
+ // because the in memory list of files is not updated until after the metadata table
+ // therefore the file is available to scans until memory is updated, but want to ensure
+ // the file is not available for garbage collection... if memory were updated
+ // before this point (like major compactions do), then the following code could wait
+ // for scans to finish like major compactions do.... used to wait for scans to finish
+ // here, but that was incorrect because a scan could start after waiting but before
+ // memory was updated... assuming the file is always in use by scans leads to
+ // one uneeded metadata update when it was not actually in use
+ Set<FileRef> filesInUseByScans = Collections.emptySet();
+ if (absMergeFile != null)
+ filesInUseByScans = Collections.singleton(absMergeFile);
+
+ // very important to write delete entries outside of log lock, because
+ // this metadata write does not go up... it goes sideways or to itself
+ if (absMergeFile != null)
+ MetadataTableUtil.addDeleteEntries(extent, Collections.singleton(absMergeFile), SystemCredentials.get());
+
+ Set<String> unusedWalLogs = beginClearingUnusedLogs();
+ try {
+ // the order of writing to metadata and walog is important in the face of machine/process failures
+ // need to write to metadata before writing to walog, when things are done in the reverse order
+ // data could be lost... the minor compaction start even should be written before the following metadata
+ // write is made
+
+ synchronized (timeLock) {
+ if (commitSession.getMaxCommittedTime() > persistedTime)
+ persistedTime = commitSession.getMaxCommittedTime();
+
+ String time = tabletTime.getMetadataValue(persistedTime);
+ MasterMetadataUtil.updateTabletDataFile(extent, newDatafile, absMergeFile, dfv, time, SystemCredentials.get(), filesInUseByScans,
+ tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, lastLocation, flushId);
+ }
+
+ } finally {
+ finishClearingUnusedLogs();
+ }
+
+ do {
+ try {
+ // the purpose of making this update use the new commit session, instead of the old one passed in,
+ // is because the new one will reference the logs used by current memory...
+
+ tabletServer.minorCompactionFinished(tabletMemory.getCommitSession(), newDatafile.toString(), commitSession.getWALogSeq() + 2);
+ break;
+ } catch (IOException e) {
+ log.error("Failed to write to write-ahead log " + e.getMessage() + " will retry", e);
+ UtilWaitThread.sleep(1 * 1000);
+ }
+ } while (true);
+
+ synchronized (Tablet.this) {
+ lastLocation = null;
+
+ t1 = System.currentTimeMillis();
+ if (datafileSizes.containsKey(newDatafile)) {
+ log.error("Adding file that is already in set " + newDatafile);
+ }
+
+ if (dfv.getNumEntries() > 0) {
+ datafileSizes.put(newDatafile, dfv);
+ }
+
+ if (absMergeFile != null) {
+ datafileSizes.remove(absMergeFile);
+ }
+
+ unreserveMergingMinorCompactionFile(absMergeFile);
+
+ dataSourceDeletions.incrementAndGet();
+ tabletMemory.finishedMinC();
+
+ lastFlushID = flushId;
+
+ computeNumEntries();
+ t2 = System.currentTimeMillis();
+ }
+
+ // must do this after list of files in memory is updated above
+ removeFilesAfterScan(filesInUseByScans);
+
+ if (absMergeFile != null)
+ log.log(TLevel.TABLET_HIST, extent + " MinC [" + absMergeFile + ",memory] -> " + newDatafile);
+ else
+ log.log(TLevel.TABLET_HIST, extent + " MinC [memory] -> " + newDatafile);
+ log.debug(String.format("MinC finish lock %.2f secs %s", (t2 - t1) / 1000.0, getExtent().toString()));
+ if (dfv.getSize() > acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD)) {
+ log.debug(String.format("Minor Compaction wrote out file larger than split threshold. split threshold = %,d file size = %,d",
+ acuTableConf.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD), dfv.getSize()));
+ }
+
+ }
+
+ public void reserveMajorCompactingFiles(Collection<FileRef> files) {
+ if (majorCompactingFiles.size() != 0)
+ throw new IllegalStateException("Major compacting files not empty " + majorCompactingFiles);
+
+ if (mergingMinorCompactionFile != null && files.contains(mergingMinorCompactionFile))
+ throw new IllegalStateException("Major compaction tried to resrve file in use by minor compaction " + mergingMinorCompactionFile);
+
+ majorCompactingFiles.addAll(files);
+ }
+
+ public void clearMajorCompactingFile() {
+ majorCompactingFiles.clear();
+ }
+
+ void bringMajorCompactionOnline(Set<FileRef> oldDatafiles, FileRef tmpDatafile, FileRef newDatafile, Long compactionId, DataFileValue dfv)
+ throws IOException {
+ long t1, t2;
+
+ if (!extent.isRootTablet()) {
+
+ if (fs.exists(newDatafile.path())) {
+ log.error("Target map file already exist " + newDatafile, new Exception());
+ throw new IllegalStateException("Target map file already exist " + newDatafile);
+ }
+
+ // rename before putting in metadata table, so files in metadata table should
+ // always exist
+ rename(fs, tmpDatafile.path(), newDatafile.path());
+
+ if (dfv.getNumEntries() == 0) {
+ fs.deleteRecursively(newDatafile.path());
+ }
+ }
+
+ TServerInstance lastLocation = null;
+ synchronized (Tablet.this) {
+
+ t1 = System.currentTimeMillis();
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ dataSourceDeletions.incrementAndGet();
+
+ if (extent.isRootTablet()) {
+
+ waitForScansToFinish(oldDatafiles, true, Long.MAX_VALUE);
+
+ try {
+ if (!zoo.isLockHeld(tabletServer.getLock().getLockID())) {
+ throw new IllegalStateException();
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
+ }
+
+ // mark files as ready for deletion, but
+ // do not delete them until we successfully
+ // rename the compacted map file, in case
+ // the system goes down
+
+ RootFiles.replaceFiles(acuTableConf, fs, location, oldDatafiles, tmpDatafile, newDatafile);
+ }
+
+ // atomically remove old files and add new file
+ for (FileRef oldDatafile : oldDatafiles) {
+ if (!datafileSizes.containsKey(oldDatafile)) {
+ log.error("file does not exist in set " + oldDatafile);
+ }
+ datafileSizes.remove(oldDatafile);
+ majorCompactingFiles.remove(oldDatafile);
+ }
+
+ if (datafileSizes.containsKey(newDatafile)) {
+ log.error("Adding file that is already in set " + newDatafile);
+ }
+
+ if (dfv.getNumEntries() > 0) {
+ datafileSizes.put(newDatafile, dfv);
+ }
+
+ // could be used by a follow on compaction in a multipass compaction
+ majorCompactingFiles.add(newDatafile);
+
+ computeNumEntries();
+
+ lastLocation = Tablet.this.lastLocation;
+ Tablet.this.lastLocation = null;
+
+ if (compactionId != null)
+ lastCompactID = compactionId;
+
+ t2 = System.currentTimeMillis();
+ }
+
+ if (!extent.isRootTablet()) {
+ Set<FileRef> filesInUseByScans = waitForScansToFinish(oldDatafiles, false, 10000);
+ if (filesInUseByScans.size() > 0)
+ log.debug("Adding scan refs to metadata " + extent + " " + filesInUseByScans);
+ MasterMetadataUtil.replaceDatafiles(extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, SystemCredentials.get(),
+ tabletServer.getClientAddressString(), lastLocation, tabletServer.getLock());
+ removeFilesAfterScan(filesInUseByScans);
+ }
+
+ log.debug(String.format("MajC finish lock %.2f secs", (t2 - t1) / 1000.0));
+ log.log(TLevel.TABLET_HIST, extent + " MajC " + oldDatafiles + " --> " + newDatafile);
+ }
+
+ public SortedMap<FileRef,DataFileValue> getDatafileSizes() {
+ synchronized (Tablet.this) {
+ TreeMap<FileRef,DataFileValue> copy = new TreeMap<FileRef,DataFileValue>(datafileSizes);
+ return Collections.unmodifiableSortedMap(copy);
+ }
+ }
+
+ public Set<FileRef> getFiles() {
+ synchronized (Tablet.this) {
+ HashSet<FileRef> files = new HashSet<FileRef>(datafileSizes.keySet());
+ return Collections.unmodifiableSet(files);
+ }
+ }
+
+ }
+
+ public Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
+ throws IOException {
+ this(tabletServer, location, extent, trm, CachedConfiguration.getInstance(), tabletsKeyValues);
+ splitCreationTime = 0;
+ }
+
+ public Tablet(KeyExtent extent, TabletServer tabletServer, TabletResourceManager trm, SplitInfo info) throws IOException {
+ this(tabletServer, new Text(info.dir), extent, trm, CachedConfiguration.getInstance(), info.datafiles, info.time, info.initFlushID, info.initCompactID, info.lastLocation);
+ splitCreationTime = System.currentTimeMillis();
+ }
+
+ /**
+ * Only visibile for testing
+ */
+ @VisibleForTesting
+ protected Tablet(TabletTime tabletTime, String tabletDirectory, int logId, Path location, DatafileManager datafileManager) {
+ this.tabletTime = tabletTime;
+ this.tabletDirectory = tabletDirectory;
+ this.logId = logId;
+ this.location = location;
+ this.datafileManager = datafileManager;
+ }
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+ this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), tabletsKeyValues);
+ }
+
+ static private final List<LogEntry> EMPTY = Collections.emptyList();
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf,
+ SortedMap<FileRef,DataFileValue> datafiles, String time, long initFlushID, long initCompactID, TServerInstance last) throws IOException {
+ this(tabletServer, location, extent, trm, conf, VolumeManagerImpl.get(), EMPTY, datafiles, time, last, new HashSet<FileRef>(), initFlushID, initCompactID);
+ }
+
+ private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ SortedMap<Key,Value> entries;
+
+ if (extent.isRootTablet()) {
+ return null;
+ } else {
+ entries = new TreeMap<Key,Value>();
+ Text rowName = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().compareRow(rowName) == 0 && TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
+ entries.put(new Key(entry.getKey()), new Value(entry.getValue()));
+ }
+ }
+ }
+
+ // log.debug("extent : "+extent+" entries : "+entries);
+
+ if (entries.size() == 1)
+ return entries.values().iterator().next().toString();
+ return null;
+ }
+
+ private static SortedMap<FileRef,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, VolumeManager fs, KeyExtent extent,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+
+ TreeMap<FileRef,DataFileValue> datafiles = new TreeMap<FileRef,DataFileValue>();
+
+ if (extent.isRootTablet()) { // the meta0 tablet
+ Path location = new Path(MetadataTableUtil.getRootTabletDir());
+
+ // cleanUpFiles() has special handling for delete. files
+ FileStatus[] files = fs.listStatus(location);
+ Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true);
+ for (String good : goodPaths) {
+ Path path = new Path(good);
+ String filename = path.getName();
+ FileRef ref = new FileRef(location.toString() + "/" + filename, path);
+ DataFileValue dfv = new DataFileValue(0, 0);
+ datafiles.put(ref, dfv);
+ }
+ } else {
+
+ Text rowName = extent.getMetadataEntry();
+
+ String tableId = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
+ ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SystemCredentials.get(), tableId, Authorizations.EMPTY);
+
+ // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
+ // reduced batch size to improve performance
+ // changed here after endKeys were implemented from 10 to 1000
+ mdScanner.setBatchSize(1000);
+
+ // leave these in, again, now using endKey for safety
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+
+ mdScanner.setRange(new Range(rowName));
+
+ for (Entry<Key,Value> entry : mdScanner) {
+
+ if (entry.getKey().compareRow(rowName) != 0) {
+ break;
+ }
+
+ FileRef ref = new FileRef(fs, entry.getKey());
+ datafiles.put(ref, new DataFileValue(entry.getValue().get()));
+ }
+ }
+ return datafiles;
+ }
+
+ private static List<LogEntry> lookupLogEntries(KeyExtent ke, SortedMap<Key,Value> tabletsKeyValues) {
+ List<LogEntry> logEntries = new ArrayList<LogEntry>();
+
+ if (ke.isMeta()) {
+ try {
+ logEntries = MetadataTableUtil.getLogEntries(SystemCredentials.get(), ke);
+ } catch (Exception ex) {
+ throw new RuntimeException("Unable to read tablet log entries", ex);
+ }
+ } else {
+ log.debug("Looking at metadata " + tabletsKeyValues);
+ Text row = ke.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row)) {
+ if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
+ logEntries.add(LogEntry.fromKeyValue(key, entry.getValue()));
+ }
+ }
+ }
+ }
+
+ log.debug("got " + logEntries + " for logs for " + ke);
+ return logEntries;
+ }
+
+ private static Set<FileRef> lookupScanFiles(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) throws IOException {
+ HashSet<FileRef> scanFiles = new HashSet<FileRef>();
+
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && key.getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
+ scanFiles.add(new FileRef(fs, key));
+ }
+ }
+
+ return scanFiles;
+ }
+
+ private static long lookupFlushID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ return Long.parseLong(entry.getValue().toString());
+ }
+
+ return -1;
+ }
+
+ private static long lookupCompactID(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ Text row = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ Key key = entry.getKey();
+ if (key.getRow().equals(row) && TabletsSection.ServerColumnFamily.COMPACT_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier()))
+ return Long.parseLong(entry.getValue().toString());
+ }
+
+ return -1;
+ }
+
+ private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, Configuration conf, VolumeManager fs,
+ SortedMap<Key,Value> tabletsKeyValues) throws IOException {
+ this(tabletServer, location, extent, trm, conf, fs, lookupLogEntries(extent, tabletsKeyValues), lookupDatafiles(tabletServer.getSystemConfiguration(), fs,
+ extent, tabletsKeyValues), lookupTime(tabletServer.getSystemConfiguration(), extent, tabletsKeyValues), lookupLastServer(extent, tabletsKeyValues),
+ lookupScanFiles(extent, tabletsKeyValues, fs), lookupFlushID(extent, tabletsKeyValues), lookupCompactID(extent, tabletsKeyValues));
+ }
+
+ private static TServerInstance lookupLastServer(KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().getColumnFamily().compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
+ return new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
+ */
+ private Tablet(final TabletServer tabletServer, final Text location, final KeyExtent extent, final TabletResourceManager trm, final Configuration conf,
+ final VolumeManager fs, final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time,
+ final TServerInstance lastLocation, Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
+
+ TabletFiles tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer.getLock(), fs, extent, new TabletFiles(location.toString(), rawLogEntries,
+ rawDatafiles));
+
+ Path locationPath;
+
+ if (tabletPaths.dir.contains(":")) {
+ locationPath = new Path(tabletPaths.dir.toString());
+ } else {
+ locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString());
+ }
+
+ final List<LogEntry> logEntries = tabletPaths.logEntries;
+ final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles;
+
+ this.location = locationPath;
+ this.lastLocation = lastLocation;
+ this.tabletDirectory = tabletPaths.dir;
+ this.conf = conf;
+ this.acuTableConf = tabletServer.getTableConfiguration(extent);
+
+ this.fs = fs;
+ this.extent = extent;
+ this.tabletResources = trm;
+
+ this.lastFlushID = initFlushID;
+ this.lastCompactID = initCompactID;
+
+ if (extent.isRootTablet()) {
+ long rtime = Long.MIN_VALUE;
+ for (FileRef ref : datafiles.keySet()) {
+ Path path = ref.path();
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
+ long maxTime = -1;
+ try {
+
+ while (reader.hasTop()) {
+ maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
+ reader.next();
+ }
+
+ } finally {
+ reader.close();
+ }
+
+ if (maxTime > rtime) {
+ time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
+ rtime = maxTime;
+ }
+ }
+ }
+ if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) {
+ // recovery... old root tablet has no data, so time doesn't matter:
+ time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE;
+ }
+
+ this.tabletServer = tabletServer;
+ this.logId = tabletServer.createLogId(extent);
+
+ this.timer = new TabletStatsKeeper();
+
+ setupDefaultSecurityLabels(extent);
+
+ tabletMemory = new TabletMemory();
+ tabletTime = TabletTime.getInstance(time);
+ persistedTime = tabletTime.getTime();
+
+ acuTableConf.addObserver(configObserver = new ConfigurationObserver() {
+
+ private void reloadConstraints() {
+ constraintChecker.set(new ConstraintChecker(acuTableConf));
+ }
+
+ @Override
+ public void propertiesChanged() {
+ reloadConstraints();
+
+ try {
+ setupDefaultSecurityLabels(extent);
+ } catch (Exception e) {
+ log.error("Failed to reload default security labels for extent: " + extent.toString());
+ }
+ }
+
+ @Override
+ public void propertyChanged(String prop) {
+ if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
+ reloadConstraints();
+ else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
+ try {
+ log.info("Default security labels changed for extent: " + extent.toString());
+ setupDefaultSecurityLabels(extent);
+ } catch (Exception e) {
+ log.error("Failed to reload default security labels for extent: " + extent.toString());
+ }
+ }
+
+ }
+
+ @Override
+ public void sessionExpired() {
+ log.debug("Session expired, no longer updating per table props...");
+ }
+
+ });
+
+ acuTableConf.getNamespaceConfiguration().addObserver(configObserver);
+
+ // Force a load of any per-table properties
+ configObserver.propertiesChanged();
+
+ tabletResources.setTablet(this, acuTableConf);
+ if (!logEntries.isEmpty()) {
+ log.info("Starting Write-Ahead Log recovery for " + this.extent);
+ // count[0] = entries used on tablet
+ // count[1] = track max time from walog entries wihtout timestamps
+ final long[] count = new long[2];
+ final CommitSession commitSession = tabletMemory.getCommitSession();
+ count[1] = Long.MIN_VALUE;
+ try {
+ Set<String> absPaths = new HashSet<String>();
+ for (FileRef ref : datafiles.keySet())
+ absPaths.add(ref.path().toString());
+
+ tabletServer.recover(this.tabletServer.getFileSystem(), this, logEntries, absPaths, new MutationReceiver() {
+ @Override
+ public void receive(Mutation m) {
+ // LogReader.printMutation(m);
+ Collection<ColumnUpdate> muts = m.getUpdates();
+ for (ColumnUpdate columnUpdate : muts) {
+ if (!columnUpdate.hasTimestamp()) {
+ // if it is not a user set timestamp, it must have been set
+ // by the system
+ count[1] = Math.max(count[1], columnUpdate.getTimestamp());
+ }
+ }
+ tabletMemory.mutate(commitSession, Collections.singletonList(m));
+ count[0]++;
+ }
+ });
+
+ if (count[1] != Long.MIN_VALUE) {
+ tabletTime.useMaxTimeFromWALog(count[1]);
+ }
+ commitSession.updateMaxCommittedTime(tabletTime.getTime());
+
+ if (count[0] == 0) {
+ log.debug("No replayed mutations applied, removing unused entries for " + extent);
+ MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock());
+ logEntries.clear();
+ }
+
+ } catch (Throwable t) {
+ if (acuTableConf.getBoolean(Property.TABLE_FAILURES_IGNORE)) {
+ log.warn("Error recovering from log files: ", t);
+ } else {
+ throw new RuntimeException(t);
+ }
+ }
+ // make some closed references that represent the recovered logs
+ currentLogs = new HashSet<DfsLogger>();
+ for (LogEntry logEntry : logEntries) {
+ for (String log : logEntry.logSet) {
+ currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), log, logEntry.getColumnQualifier().toString()));
+ }
+ }
+
+ log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries()
+ + " entries created)");
+ }
+
+ String contextName = acuTableConf.get(Property.TABLE_CLASSPATH);
+ if (contextName != null && !contextName.equals("")) {
+ // initialize context classloader, instead of possibly waiting for it to initialize for a scan
+ // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292
+ AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
+ }
+
+ // do this last after tablet is completely setup because it
+ // could cause major compaction to start
+ datafileManager = new DatafileManager(datafiles);
+
+ computeNumEntries();
+
+ datafileManager.removeFilesAfterScan(scanFiles);
+
+ // look for hints of a failure on the previous tablet server
+ if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
+ // look for any temp files hanging around
+ removeOldTemporaryFiles();
+ }
+
+ log.log(TLevel.TABLET_HIST, extent + " opened");
+ }
+
+ private void removeOldTemporaryFiles() {
+ // remove any temporary files created by a previous tablet server
+ try {
+ for (FileStatus tmp : fs.globStatus(new Path(location, "*_tmp"))) {
+ try {
+ log.debug("Removing old temp file " + tmp.getPath());
+ fs.delete(tmp.getPath());
+ } catch (IOException ex) {
+ log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
+ }
+ }
+ } catch (IOException ex) {
+ log.error("Error scanning for old temp files in " + location);
+ }
+ }
+
+ private void setupDefaultSecurityLabels(KeyExtent extent) {
+ if (extent.isMeta()) {
+ defaultSecurityLabel = new byte[0];
+ } else {
+ try {
+ ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
+ this.defaultSecurityLabel = cv.getExpression();
+ } catch (Exception e) {
+ log.error(e, e);
+ this.defaultSecurityLabel = new byte[0];
+ }
+ }
+ }
+
+ public static class KVEntry extends KeyValue {
+ private static final long serialVersionUID = 1L;
+
+ public KVEntry(Key k, Value v) {
+ super(new Key(k), Arrays.copyOf(v.get(), v.get().length));
+ }
+
+ int numBytes() {
+ return getKey().getSize() + getValue().get().length;
+ }
+
+ int estimateMemoryUsed() {
+ return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object
+ }
+ }
+
+ private LookupResult lookup(SortedKeyValueIterator<Key,Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, ArrayList<KVEntry> results,
+ long maxResultsSize) throws IOException {
+
+ LookupResult lookupResult = new LookupResult();
+
+ boolean exceededMemoryUsage = false;
+ boolean tabletClosed = false;
+
+ Set<ByteSequence> cfset = null;
+ if (columnSet.size() > 0)
+ cfset = LocalityGroupUtil.families(columnSet);
+
+ for (Range range : ranges) {
+
+ if (exceededMemoryUsage || tabletClosed) {
+ lookupResult.unfinishedRanges.add(range);
+ continue;
+ }
+
+ int entriesAdded = 0;
+
+ try {
+ if (cfset != null)
+ mmfi.seek(range, cfset, true);
+ else
+ mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
+
+ while (mmfi.hasTop()) {
+ Key key = mmfi.getTopKey();
+
+ KVEntry kve = new KVEntry(key, mmfi.getTopValue());
+ results.add(kve);
+ entriesAdded++;
+ lookupResult.bytesAdded += kve.estimateMemoryUsed();
+ lookupResult.dataSize += kve.numBytes();
+
+ exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
+
+ if (exceededMemoryUsage) {
+ addUnfinishedRange(lookupResult, range, key, false);
+ break;
+ }
+
+ mmfi.next();
+ }
+
+ } catch (TooManyFilesException tmfe) {
+ // treat this as a closed tablet, and let the client retry
+ log.warn("Tablet " + getExtent() + " has too many files, batch lookup can not run");
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } catch (IOException ioe) {
+ if (shutdownInProgress()) {
+ // assume HDFS shutdown hook caused this exception
+ log.debug("IOException while shutdown in progress ", ioe);
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw ioe;
+ }
+ } catch (IterationInterruptedException iie) {
+ if (isClosed()) {
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw iie;
+ }
+ } catch (TabletClosedException tce) {
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
+ tabletClosed = true;
+ }
+
+ }
+
+ return lookupResult;
+ }
+
+ private void handleTabletClosedDuringScan(ArrayList<KVEntry> results, LookupResult lookupResult, boolean exceededMemoryUsage, Range range, int entriesAdded) {
+ if (exceededMemoryUsage)
+ throw new IllegalStateException("tablet should not exceed memory usage or close, not both");
+
+ if (entriesAdded > 0)
+ addUnfinishedRange(lookupResult, range, results.get(results.size() - 1).getKey(), false);
+ else
+ lookupResult.unfinishedRanges.add(range);
+
+ lookupResult.closed = true;
+ }
+
+ private void addUnfinishedRange(LookupResult lookupResult, Range range, Key key, boolean inclusiveStartKey) {
+ if (range.getEndKey() == null || key.compareTo(range.getEndKey()) < 0) {
+ Range nlur = new Range(new Key(key), inclusiveStartKey, range.getEndKey(), range.isEndKeyInclusive());
+ lookupResult.unfinishedRanges.add(nlur);
+ }
+ }
+
+ public static interface KVReceiver {
+ void receive(List<KVEntry> matches) throws IOException;
+ }
+
+ class LookupResult {
+ List<Range> unfinishedRanges = new ArrayList<Range>();
+ long bytesAdded = 0;
+ long dataSize = 0;
+ boolean closed = false;
+ }
+
+ public LookupResult lookup(List<Range> ranges, HashSet<Column> columns, Authorizations authorizations, ArrayList<KVEntry> results, long maxResultSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag) throws IOException {
+
+ if (ranges.size() == 0) {
+ return new LookupResult();
+ }
+
+ ranges = Range.mergeOverlapping(ranges);
+ Collections.sort(ranges);
+
+ Range tabletRange = extent.toDataRange();
+ for (Range range : ranges) {
+ // do a test to see if this range falls within the tablet, if it does not
+ // then clip will throw an exception
+ tabletRange.clip(range);
+ }
+
+ ScanDataSource dataSource = new ScanDataSource(authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
+
+ LookupResult result = null;
+
+ try {
+ SortedKeyValueIterator<Key,Value> iter = new SourceSwitchingIterator(dataSource);
+ result = lookup(iter, ranges, columns, results, maxResultSize);
+ return result;
+ } catch (IOException ioe) {
+ dataSource.close(true);
+ throw ioe;
+ } finally {
+ // code in finally block because always want
+ // to return mapfiles, even when exception is thrown
+ dataSource.close(false);
+
+ synchronized (this) {
+ queryCount += results.size();
+ if (result != null)
+ queryBytes += result.dataSize;
+ }
+ }
+ }
+
+ private Batch nextBatch(SortedKeyValueIterator<Key,Value> iter, Range range, int num, Set<Column> columns) throws IOException {
+
+ // log.info("In nextBatch..");
+
+ List<KVEntry> results = new ArrayList<KVEntry>();
+ Key key = null;
+
+ Value value;
+ long resultSize = 0L;
+ long resultBytes = 0L;
+
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ if (columns.size() == 0) {
+ iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
+ } else {
+ iter.seek(range, LocalityGroupUtil.families(columns), true);
+ }
+
+ Key continueKey = null;
+ boolean skipContinueKey = false;
+
+ boolean endOfTabletReached = false;
+ while (iter.hasTop()) {
+
+ value = iter.getTopValue();
+ key = iter.getTopKey();
+
+ KVEntry kvEntry = new KVEntry(key, value); // copies key and value
+ results.add(kvEntry);
+ resultSize += kvEntry.estimateMemoryUsed();
+ resultBytes += kvEntry.numBytes();
+
+ if (resultSize >= maxResultsSize || results.size() >= num) {
+ continueKey = new Key(key);
+ skipContinueKey = true;
+ break;
+ }
+
+ iter.next();
+ }
+
+ if (iter.hasTop() == false) {
+ endOfTabletReached = true;
+ }
+
+ Batch retBatch = new Batch();
+ retBatch.numBytes = resultBytes;
+
+ if (!endOfTabletReached) {
+ retBatch.continueKey = continueKey;
+ retBatch.skipContinueKey = skipContinueKey;
+ } else {
+ retBatch.continueKey = null;
+ }
+
+ if (endOfTabletReached && results.size() == 0)
+ retBatch.results = null;
+ else
+ retBatch.results = results;
+
+ return retBatch;
+ }
+
+ /**
+ * Determine if a JVM shutdown is in progress.
+ *
+ */
+ private boolean shutdownInProgress() {
+ try {
+ Runtime.getRuntime().removeShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {}
+ }));
+ } catch (IllegalStateException ise) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private class Batch {
+ public boolean skipContinueKey;
+ public List<KVEntry> results;
+ public Key continueKey;
+ public long numBytes;
+ }
+
+ Scanner createScanner(Range range, int num, Set<Column> columns, Authorizations authorizations, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
+ boolean isolated, AtomicBoolean interruptFlag) {
+ // do a test to see if this range falls within the tablet, if it does not
+ // then clip will throw an exception
+ extent.toDataRange().clip(range);
+
+ ScanOptions opts = new ScanOptions(num, authorizations, this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag, isolated);
+ return new Scanner(range, opts);
+ }
+
+ class ScanBatch {
+ boolean more;
+ List<KVEntry> results;
+
+ ScanBatch(List<KVEntry> results, boolean more) {
+ this.results = results;
+ this.more = more;
+ }
+ }
+
+ class Scanner {
+
+ private ScanOptions options;
+ private Range range;
+ private SortedKeyValueIterator<Key,Value> isolatedIter;
+ private ScanDataSource isolatedDataSource;
+ private boolean sawException = false;
+ private boolean scanClosed = false;
+
+ Scanner(Range range, ScanOptions options) {
+ this.range = range;
+ this.options = options;
+ }
+
+ synchronized ScanBatch read() throws IOException, TabletClosedException {
+
+ if (sawException)
+ throw new IllegalStateException("Tried to use scanner after exception occurred.");
+
+ if (scanClosed)
+ throw new IllegalStateException("Tried to use scanner after it was closed.");
+
+ Batch results = null;
+
+ ScanDataSource dataSource;
+
+ if (options.isolated) {
+ if (isolatedDataSource == null)
+ isolatedDataSource = new ScanDataSource(options);
+ dataSource = isolatedDataSource;
+ } else {
+ dataSource = new ScanDataSource(options);
+ }
+
+ try {
+
+ SortedKeyValueIterator<Key,Value> iter;
+
+ if (options.isolated) {
+ if (isolatedIter == null)
+ isolatedIter = new SourceSwitchingIterator(dataSource, true);
+ else
+ isolatedDataSource.fileManager.reattach();
+ iter = isolatedIter;
+ } else {
+ iter = new SourceSwitchingIterator(dataSource, false);
+ }
+
+ results = nextBatch(iter, range, options.num, options.columnSet);
+
+ if (results.results == null) {
+ range = null;
+ return new ScanBatch(new ArrayList<Tablet.KVEntry>(), false);
+ } else if (results.continueKey == null) {
+ return new ScanBatch(results.results, false);
+ } else {
+ range = new Range(results.continueKey, !results.skipContinueKey, range.getEndKey(), range.isEndKeyInclusive());
+ return new ScanBatch(results.results, true);
+ }
+
+ } catch (IterationInterruptedException iie) {
+ sawException = true;
+ if (isClosed())
+ throw new TabletClosedException(iie);
+ else
+ throw iie;
+ } catch (IOException ioe) {
+ if (shutdownInProgress()) {
+ log.debug("IOException while shutdown in progress ", ioe);
+ throw new TabletClosedException(ioe); // assume IOException was caused by execution of HDFS shutdown hook
+ }
+
+ sawException = true;
+ dataSource.close(true);
+ throw ioe;
+ } catch (RuntimeException re) {
+ sawException = true;
+ throw re;
+ } finally {
+ // code in finally block because always want
+ // to return mapfiles, even when exception is thrown
+ if (!options.isolated)
+ dataSource.close(false);
+ else if (dataSource.fileManager != null)
+ dataSource.fileManager.detach();
+
+ synchronized (Tablet.this) {
+ if (results != null && results.results != null) {
+ long more = results.results.size();
+ queryCount += more;
+ queryBytes += results.numBytes;
+ }
+ }
+ }
+ }
+
+ // close and read are synchronized because can not call close on the data source while it is in use
+ // this cloud lead to the case where file iterators that are in use by a thread are returned
+ // to the pool... this would be bad
+ void close() {
+ options.interruptFlag.set(true);
+ synchronized (this) {
+ scanClosed = true;
+ if (isolatedDataSource != null)
+ isolatedDataSource.close(false);
+ }
+ }
+ }
+
+ static class ScanOptions {
+
+ // scan options
+ Authorizations authorizations;
+ byte[] defaultLabels;
+ Set<Column> columnSet;
+ List<IterInfo> ssiList;
+ Map<String,Map<String,String>> ssio;
+ AtomicBoolean interruptFlag;
+ int num;
+ boolean isolated;
+
+ ScanOptions(int num, Authorizations authorizations, byte[] defaultLabels, Set<Column> columnSet, List<IterInfo> ssiList,
+ Map<String,Map<String,String>> ssio, AtomicBoolean interruptFlag, boolean isolated) {
+ this.num = num;
+ this.authorizations = authorizations;
+ this.defaultLabels = defaultLabels;
+ this.columnSet = columnSet;
+ this.ssiList = ssiList;
+ this.ssio = ssio;
+ this.interruptFlag = interruptFlag;
+ this.isolated = isolated;
+ }
+
+ }
+
+ class ScanDataSource implements DataSource {
+
+ // data source state
+ private ScanFileManager fileManager;
+ private SortedKeyValueIterator<Key,Value> iter;
+ private long expectedDeletionCount;
+ private List<MemoryIterator> memIters = null;
+ private long fileReservationId;
+ private AtomicBoolean interruptFlag;
+ private StatsIterator statsIterator;
+
+ ScanOptions options;
+
+ ScanDataSource(Authorizations authorizations, byte[] defaultLabels, HashSet<Column> columnSet, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
+ AtomicBoolean interruptFlag) {
+ expectedDeletionCount = dataSourceDeletions.get();
+ this.options = new ScanOptions(-1, authorizations, defaultLabels, columnSet, ssiList, ssio, interruptFlag, false);
+ this.interruptFlag = interruptFlag;
+ }
+
+ ScanDataSource(ScanOptions options) {
+ expectedDeletionCount = dataSourceDeletions.get();
+ this.options = options;
+ this.interruptFlag = options.interruptFlag;
+ }
+
+ @Override
+ public DataSource getNewDataSource() {
+ if (!isCurrent()) {
+ // log.debug("Switching data sources during a scan");
+ if (memIters != null) {
+ tabletMemory.returnIterators(memIters);
+ memIters = null;
+ datafileManager.returnFilesForScan(fileReservationId);
+ fileReservationId = -1;
+ }
+
+ if (fileManager != null)
+ fileManager.releaseOpenFiles(false);
+
+ expectedDeletionCount = dataSourceDeletions.get();
+ iter = null;
+
+ return this;
+ } else
+ return this;
+ }
+
+ @Override
+ public boolean isCurrent() {
+ return expectedDeletionCount == dataSourceDeletions.get();
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> iterator() throws IOException {
+ if (iter == null)
+ iter = createIterator();
+ return iter;
+ }
+
+ private SortedKeyValueIterator<Key,Value> createIterator() throws IOException {
+
+ Map<FileRef,DataFileValue> files;
+
+ synchronized (Tablet.this) {
+
+ if (memIters != null)
+ throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
+
+ if (Tablet.this.closed)
+ throw new TabletClosedException();
+
+ if (interruptFlag.get())
+ throw new IterationInterruptedException(extent.toString() + " " + interruptFlag.hashCode());
+
+ // only acquire the file manager when we know the tablet is open
+ if (fileManager == null) {
+ fileManager = tabletResources.newScanFileManager();
+ activeScans.add(this);
+ }
+
+ if (fileManager.getNumOpenFiles() != 0)
+ throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
+
+ // set this before trying to get iterators in case
+ // getIterators() throws an exception
+ expectedDeletionCount = dataSourceDeletions.get();
+
+ memIters = tabletMemory.getIterators();
+ Pair<Long,Map<FileRef,DataFileValue>> reservation = datafileManager.reserveFilesForScan();
+ fileReservationId = reservation.getFirst();
+ files = reservation.getSecond();
+ }
+
+ Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, options.isolated);
+
+ List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(mapfiles.size() + memIters.size());
+
+ iters.addAll(mapfiles);
+ iters.addAll(memIters);
+
+ for (SortedKeyValueIterator<Key,Value> skvi : iters)
+ ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
+
+ MultiIterator multiIter = new MultiIterator(iters, extent);
+
+ TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager, files);
+
+ statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, scannedCount);
+
+ DeletingIterator delIter = new DeletingIterator(statsIterator, false);
+
+ ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
+
+ ColumnQualifierFilter colFilter = new ColumnQualifierFilter(cfsi, options.columnSet);
+
+ VisibilityFilter visFilter = new VisibilityFilter(colFilter, options.authorizations, options.defaultLabels);
+
+ return iterEnv.getTopLevelIterator(IteratorUtil
+ .loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.ssiList, options.ssio, iterEnv));
+ }
+
+ private void close(boolean sawErrors) {
+
+ if (memIters != null) {
+ tabletMemory.returnIterators(memIters);
+ memIters = null;
+ datafileManager.returnFilesForScan(fileReservationId);
+ fileReservationId = -1;
+ }
+
+ synchronized (Tablet.this) {
+ activeScans.remove(this);
+ if (activeScans.size() == 0)
+ Tablet.this.notifyAll();
+ }
+
+ if (fileManager != null) {
+ fileManager.releaseOpenFiles(sawErrors);
+ fileManager = null;
+ }
+
+ if (statsIterator != null) {
+ statsIterator.report();
+ }
+
+ }
+
+ public void interrupt() {
+ interruptFlag.set(true);
+ }
+
+ @Override
+ public DataSource getDeepCopyDataSource(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private DataFileValue minorCompact(Configuration conf, VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile,
+ boolean hasQueueTime, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
+ boolean failed = false;
+ long start = System.currentTimeMillis();
+ timer.incrementStatusMinor();
+
+ long count = 0;
+
+ try {
+ Span span = Trace.start("write");
+ CompactionStats stats;
+ try {
+ count = memTable.getNumEntries();
+
+ DataFileValue dfv = null;
+ if (mergeFile != null)
+ dfv = datafileManager.getDatafileSizes().get(mergeFile);
+
+ MinorCompactor compactor = new MinorCompactor(conf, fs, memTable, mergeFile, dfv, tmpDatafile, acuTableConf, extent, mincReason);
+ stats = compactor.call();
+ } finally {
+ span.stop();
+ }
+ span = Trace.start("bringOnline");
+ try {
+ datafileManager.bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()),
+ commitSession, flushId);
+ } finally {
+ span.stop();
+ }
+ return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
+ } catch (Exception E) {
+ failed = true;
+ throw new RuntimeException(E);
+ } catch (Error E) {
+ // Weird errors like "OutOfMemoryError" when trying to create the thread for the compaction
+ failed = true;
+ throw new RuntimeException(E);
+ } finally {
+ try {
+ tabletMemory.finalizeMinC();
+ } catch (Throwable t) {
+ log.error("Failed to free tablet memory", t);
+ }
+
+ if (!failed) {
+ lastMinorCompactionFinishTime = System.currentTimeMillis();
+ }
+ if (tabletServer.mincMetrics.isEnabled())
+ tabletServer.mincMetrics.add(TabletServerMinCMetrics.minc, (lastMinorCompactionFinishTime - start));
+ if (hasQueueTime) {
+ timer.updateTime(Operation.MINOR, queued, start, count, failed);
+ if (tabletServer.mincMetrics.isEnabled())
+ tabletServer.mincMetrics.add(TabletServerMinCMetrics.queue, (start - queued));
+ } else
+ timer.updateTime(Operation.MINOR, start, count, failed);
+ }
+ }
+
+ private class MinorCompactionTask implements Runnable {
+
+ private long queued;
+ private CommitSession commitSession;
+ private DataFileValue stats;
+ private FileRef mergeFile;
+ private long flushId;
+ private MinorCompactionReason mincReason;
+
+ MinorCompactionTask(FileRef mergeFile, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
+ queued = System.currentTimeMillis();
+ minorCompactionWaitingToStart = true;
+ this.commitSession = commitSession;
+ this.mergeFile = mergeFile;
+ this.flushId = flushId;
+ this.mincReason = mincReason;
+ }
+
+ @Override
+ public void run() {
+ minorCompactionWaitingToStart = false;
+ minorCompactionInProgress = true;
+ Span minorCompaction = Trace.on("minorCompaction");
+ try {
+ FileRef newMapfileLocation = getNextMapFilename(mergeFile == null ? "F" : "M");
+ FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp");
+ Span span = Trace.start("waitForCommits");
+ synchronized (Tablet.this) {
+ commitSession.waitForCommitsToFinish();
+ }
+ span.stop();
+ span = Trace.start("start");
+ while (true) {
+ try {
+ // the purpose of the minor compaction start event is to keep track of the filename... in the case
+ // where the metadata table write for the minor compaction finishes and the process dies before
+ // writing the minor compaction finish event, then the start event+filename in metadata table will
+ // prevent recovery of duplicate data... the minor compaction start event could be written at any time
+ // before the metadata write for the minor compaction
+ tabletServer.minorCompactionStarted(commitSession, commitSession.getWALogSeq() + 1, newMapfileLocation.path().toString());
+ break;
+ } catch (IOException e) {
+ log.warn("Failed to write to write ahead log " + e.getMessage(), e);
+ }
+ }
+ span.stop();
+ span = Trace.start("compact");
+ this.stats = minorCompact(conf, fs, tabletMemory.getMinCMemTable(), tmpFileRef, newMapfileLocation, mergeFile, true, queued, commitSession, flushId,
+ mincReason);
+ span.stop();
+
+ if (needsSplit()) {
+ tabletServer.executeSplit(Tablet.this);
+ } else {
+ initiateMajorCompaction(MajorCompactionReason.NORMAL);
+ }
+ } catch (Throwable t) {
+ log.error("Unknown error during minor compaction for extent: " + getExtent(), t);
+ throw new RuntimeException(t);
+ } finally {
+ minorCompactionInProgress = false;
+ minorCompaction.data("extent", extent.toString());
+ minorCompaction.data("numEntries", Long.toString(this.stats.getNumEntries()));
+ minorCompaction.data("size", Long.toString(this.stats.getSize()));
+ minorCompaction.stop();
+ }
+ }
+ }
+
+ private synchronized MinorCompactionTask prepareForMinC(long flushId, MinorCompactionReason mincReason) {
+ CommitSession oldCommitSession = tabletMemory.prepareForMinC();
+ otherLogs = currentLogs;
+ currentLogs = new HashSet<DfsLogger>();
+
+ FileRef mergeFile = null;
+ if (mincReason != MinorCompactionReason.RECOVERY) {
+ mergeFile = datafileManager.reserveMergingMinorCompactionFile();
+ }
+
+ return new MinorCompactionTask(mergeFile, oldCommitSession, flushId, mincReason);
+
+ }
+
+ void flush(long tableFlushID) {
+ boolean updateMetadata = false;
+ boolean initiateMinor = false;
+
+ try {
+
+ synchronized (this) {
+
+ // only want one thing at a time to update flush ID to ensure that metadata table and tablet in memory state are consistent
+ if (updatingFlushID)
+ return;
+
+ if (lastFlushID >= tableFlushID)
+ return;
+
+ if (closing || closed || tabletMemory.memoryReservedForMinC())
+ return;
+
+ if (tabletMemory.getMemTable().getNumEntries() == 0) {
+ lastFlushID = tableFlushID;
+ updatingFlushID = true;
+ updateMetadata = true;
+ } else
+ initiateMinor = true;
+ }
+
+ if (updateMetadata) {
+ Credentials creds = SystemCredentials.get();
+ // if multiple threads were allowed to update this outside of a sync block, then it would be
+ // a race condition
+ MetadataTableUtil.updateTabletFlushID(extent, tableFlushID, creds, tabletServer.getLock());
+ } else if (initiateMinor)
+ initiateMinorCompaction(tableFlushID, MinorCompactionReason.USER);
+
+ } finally {
+ if (updateMetadata) {
+ synchronized (this) {
+ updatingFlushID = false;
+ this.notifyAll();
+ }
+ }
+ }
+
+ }
+
+ boolean initiateMinorCompaction(MinorCompactionReason mincReason) {
+ if (isClosed()) {
+ // don't bother trying to get flush id if closed... could be closed after this check but that is ok... just trying to cut down on uneeded log messages....
+ return false;
+ }
+
+ // get the flush id before the new memmap is made available for write
+ long flushId;
+ try {
+ flushId = getFlushID();
+ } catch (NoNodeException e) {
+ log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
+ return false;
+ }
+ return initiateMinorCompaction(flushId, mincReason);
+ }
+
+ boolean minorCompactNow(MinorCompactionReason mincReason) {
+ long flushId;
+ try {
+ flushId = getFlushID();
+ } catch (NoNodeException e) {
+ log.info("Asked to initiate MinC when there was no flush id " + getExtent() + " " + e.getMessage());
+ return false;
+ }
+ MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
+ if (mct == null)
+ return false;
+ mct.run();
+ return true;
+ }
+
+ boolean initiateMinorCompaction(long flushId, MinorCompactionReason mincReason) {
+ MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
+ if (mct == null)
+ return false;
+ tabletResources.executeMinorCompaction(mct);
+ return true;
+ }
+
+ private MinorCompactionTask createMinorCompactionTask(long flushId, MinorCompactionReason mincReason) {
+ MinorCompactionTask mct;
+ long t1, t2;
+
+ StringBuilder logMessage = null;
+
+ try {
+ synchronized (this) {
+ t1 = System.currentTimeMillis();
+
+ if (closing || closed || majorCompactionWaitingToStart || tabletMemory.memoryReservedForMinC() || tabletMemory.getMemTable().getNumEntries() == 0
+ || updatingFlushID) {
+
+ logMessage = new StringBuilder();
+
+ logMessage.append(extent.toString());
+ logMessage.append(" closing " + closing);
+ logMessage.append(" closed " + closed);
+ logMessage.append(" majorCompactionWaitingToStart " + majorCompactionWaitingToStart);
+ if (tabletMemory != null)
+ logMessage.append(" tabletMemory.memoryReservedForMinC() " + tabletMemory.memoryReservedForMinC());
+ if (tabletMemory != null && tabletMemory.getMemTable() != null)
+ logMessage.append(" tabletMemory.getMemTable().getNumEntries() " + tabletMemory.getMemTable().getNumEntries());
+ logMessage.append(" updatingFlushID " + updatingFlushID);
+
+ return null;
+ }
+
+ mct = prepareForMinC(flushId, mincReason);
+ t2 = System.currentTimeMillis();
+ }
+ } finally {
+ // log outside of sync block
+ if (logMessage != null && log.isDebugEnabled())
+ log.debug(logMessage);
+ }
+
+ log.debug(String.format("MinC initiate lock %.2f secs", (t2 - t1) / 1000.0));
+ return mct;
+ }
+
+ long getFlushID() throws NoNodeException {
+ try {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_FLUSH_ID;
- return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++ return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), UTF_8));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (NumberFormatException nfe) {
+ throw new RuntimeException(nfe);
+ } catch (KeeperException ke) {
+ if (ke instanceof NoNodeException) {
+ throw (NoNodeException) ke;
+ } else {
+ throw new RuntimeException(ke);
+ }
+ }
+ }
+
+ long getCompactionCancelID() {
+ String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
+ + Constants.ZTABLE_COMPACT_CANCEL_ID;
+
+ try {
- return Long.parseLong(new String(ZooReaderWriter.getInstance().getData(zTablePath, null), Constants.UTF8));
++ return Long.parseLong(new String(ZooReaderWriter.getInstance().get
<TRUNCATED>
[12/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 03635cc,0000000..9cd02eb
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -1,1051 -1,0 +1,1053 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.BatchWriterImpl;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Writer;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * provides a reference to the metadata table for updates by tablet servers
+ */
+public class MetadataTableUtil {
+
+ private static final Text EMPTY_TEXT = new Text();
+ private static final byte[] EMPTY_BYTES = new byte[0];
+ private static Map<Credentials,Writer> root_tables = new HashMap<Credentials,Writer>();
+ private static Map<Credentials,Writer> metadata_tables = new HashMap<Credentials,Writer>();
+ private static final Logger log = Logger.getLogger(MetadataTableUtil.class);
+
+ private MetadataTableUtil() {}
+
+ public synchronized static Writer getMetadataTable(Credentials credentials) {
+ Writer metadataTable = metadata_tables.get(credentials);
+ if (metadataTable == null) {
+ metadataTable = new Writer(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID);
+ metadata_tables.put(credentials, metadataTable);
+ }
+ return metadataTable;
+ }
+
+ private synchronized static Writer getRootTable(Credentials credentials) {
+ Writer rootTable = root_tables.get(credentials);
+ if (rootTable == null) {
+ rootTable = new Writer(HdfsZooInstance.getInstance(), credentials, RootTable.ID);
+ root_tables.put(credentials, rootTable);
+ }
+ return rootTable;
+ }
+
+ private static void putLockID(ZooLock zooLock, Mutation m) {
+ TabletsSection.ServerColumnFamily.LOCK_COLUMN.put(m, new Value(zooLock.getLockID().serialize(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + "/")
- .getBytes(Constants.UTF8)));
++ .getBytes(UTF_8)));
+ }
+
+ private static void update(Credentials credentials, Mutation m, KeyExtent extent) {
+ update(credentials, null, m, extent);
+ }
+
+ public static void update(Credentials credentials, ZooLock zooLock, Mutation m, KeyExtent extent) {
+ Writer t = extent.isMeta() ? getRootTable(credentials) : getMetadataTable(credentials);
+ update(t, credentials, zooLock, m);
+ }
+
+ public static void update(Writer t, Credentials credentials, ZooLock zooLock, Mutation m) {
+ if (zooLock != null)
+ putLockID(zooLock, m);
+ while (true) {
+ try {
+ t.update(m);
+ return;
+ } catch (AccumuloException e) {
+ log.error(e, e);
+ } catch (AccumuloSecurityException e) {
+ log.error(e, e);
+ } catch (ConstraintViolationException e) {
+ log.error(e, e);
+ // retrying when a CVE occurs is probably futile and can cause problems, see ACCUMULO-3096
+ throw new RuntimeException(e);
+ } catch (TableNotFoundException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+ public static void updateTabletFlushID(KeyExtent extent, long flushID, Credentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value((flushID + "").getBytes(UTF_8)));
+ update(credentials, zooLock, m, extent);
+ }
+ }
+
+ public static void updateTabletCompactID(KeyExtent extent, long compactID, Credentials credentials, ZooLock zooLock) {
+ if (!extent.isRootTablet()) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value((compactID + "").getBytes(UTF_8)));
+ update(credentials, zooLock, m, extent);
+ }
+ }
+
+ public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, Credentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- byte[] tidBytes = Long.toString(tid).getBytes(Constants.UTF8);
++ byte[] tidBytes = Long.toString(tid).getBytes(UTF_8);
+
+ for (Entry<FileRef,DataFileValue> entry : estSizes.entrySet()) {
+ Text file = entry.getKey().meta();
+ m.put(DataFileColumnFamily.NAME, file, new Value(entry.getValue().encode()));
+ m.put(TabletsSection.BulkFileColumnFamily.NAME, file, new Value(tidBytes));
+ }
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
+ update(credentials, zooLock, m, extent);
+ }
+
+ public static void updateTabletDir(KeyExtent extent, String newDir, Credentials creds, ZooLock lock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
+ update(creds, lock, m, extent);
+ }
+
+ public static void addTablet(KeyExtent extent, String path, Credentials credentials, char timeType, ZooLock lock) {
+ Mutation m = extent.getPrevRowUpdateMutation();
+
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(Constants.UTF8)));
- TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
++ TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value((timeType + "0").getBytes(UTF_8)));
+
+ update(credentials, lock, m, extent);
+ }
+
+ public static void updateTabletPrevEndRow(KeyExtent extent, Credentials credentials) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+ update(credentials, m, extent);
+ }
+
+ public static void updateTabletVolumes(KeyExtent extent, List<LogEntry> logsToRemove, List<LogEntry> logsToAdd, List<FileRef> filesToRemove,
+ SortedMap<FileRef,DataFileValue> filesToAdd, String newDir, ZooLock zooLock, Credentials credentials) {
+
+ if (extent.isRootTablet()) {
+ if (newDir != null)
+ throw new IllegalArgumentException("newDir not expected for " + extent);
+
+ if (filesToRemove.size() != 0 || filesToAdd.size() != 0)
+ throw new IllegalArgumentException("files not expected for " + extent);
+
+ // add before removing in case of process death
+ for (LogEntry logEntry : logsToAdd)
+ addLogEntry(credentials, logEntry, zooLock);
+
+ removeUnusedWALEntries(extent, logsToRemove, zooLock);
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+
+ for (LogEntry logEntry : logsToRemove)
+ m.putDelete(logEntry.getColumnFamily(), logEntry.getColumnQualifier());
+
+ for (LogEntry logEntry : logsToAdd)
+ m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
+
+ for (FileRef fileRef : filesToRemove)
+ m.putDelete(DataFileColumnFamily.NAME, fileRef.meta());
+
+ for (Entry<FileRef,DataFileValue> entry : filesToAdd.entrySet())
+ m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
+
+ if (newDir != null)
- ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(Constants.UTF8)));
++ ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(newDir.getBytes(UTF_8)));
+
+ update(credentials, m, extent);
+ }
+ }
+
+ public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, Credentials credentials) throws IOException {
+ TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
+
+ Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
+ mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ Text row = extent.getMetadataEntry();
+ VolumeManager fs = VolumeManagerImpl.get();
+
+ Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text(""));
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+
+ mdScanner.setRange(new Range(new Key(row), endKey));
+ for (Entry<Key,Value> entry : mdScanner) {
+
+ if (!entry.getKey().getRow().equals(row))
+ break;
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(new FileRef(fs, entry.getKey()), dfv);
+ }
+
+ return sizes;
+ }
+
+ public static void rollBackSplit(Text metadataEntry, Text oldPrevEndRow, Credentials credentials, ZooLock zooLock) {
+ KeyExtent ke = new KeyExtent(metadataEntry, oldPrevEndRow);
+ Mutation m = ke.getPrevRowUpdateMutation();
+ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
+ update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
+ }
+
+ public static void splitTablet(KeyExtent extent, Text oldPrevEndRow, double splitRatio, Credentials credentials, ZooLock zooLock) {
+ Mutation m = extent.getPrevRowUpdateMutation(); //
+
- TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(Constants.UTF8)));
++ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value(Double.toString(splitRatio).getBytes(UTF_8)));
+
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(oldPrevEndRow));
+ ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
+ update(credentials, zooLock, m, extent);
+ }
+
+ public static void finishSplit(Text metadataEntry, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
+ ZooLock zooLock) {
+ Mutation m = new Mutation(metadataEntry);
+ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
+ ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
+
+ for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
+ }
+
+ for (FileRef pathToRemove : highDatafilesToRemove) {
+ m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta());
+ }
+
+ update(credentials, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
+ }
+
+ public static void finishSplit(KeyExtent extent, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, Credentials credentials,
+ ZooLock zooLock) {
+ finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, credentials, zooLock);
+ }
+
+ public static void addDeleteEntries(KeyExtent extent, Set<FileRef> datafilesToDelete, Credentials credentials) throws IOException {
+
+ String tableId = extent.getTableId().toString();
+
+ // TODO could use batch writer,would need to handle failure and retry like update does - ACCUMULO-1294
+ for (FileRef pathToRemove : datafilesToDelete) {
+ update(credentials, createDeleteMutation(tableId, pathToRemove.path().toString()), extent);
+ }
+ }
+
+ public static void addDeleteEntry(String tableId, String path) throws IOException {
+ update(SystemCredentials.get(), createDeleteMutation(tableId, path), new KeyExtent(new Text(tableId), null, null));
+ }
+
+ public static Mutation createDeleteMutation(String tableId, String pathToRemove) throws IOException {
+ Path path = VolumeManagerImpl.get().getFullPath(tableId, pathToRemove);
+ Mutation delFlag = new Mutation(new Text(MetadataSchema.DeletesSection.getRowPrefix() + path.toString()));
+ delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+ return delFlag;
+ }
+
+ public static void removeScanFiles(KeyExtent extent, Set<FileRef> scanFiles, Credentials credentials, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+
+ for (FileRef pathToRemove : scanFiles)
+ m.putDelete(ScanFileColumnFamily.NAME, pathToRemove.meta());
+
+ update(credentials, zooLock, m, extent);
+ }
+
+ public static void splitDatafiles(Text table, Text midRow, double splitRatio, Map<FileRef,FileUtil.FileInfo> firstAndLastRows,
+ SortedMap<FileRef,DataFileValue> datafiles, SortedMap<FileRef,DataFileValue> lowDatafileSizes, SortedMap<FileRef,DataFileValue> highDatafileSizes,
+ List<FileRef> highDatafilesToRemove) {
+
+ for (Entry<FileRef,DataFileValue> entry : datafiles.entrySet()) {
+
+ Text firstRow = null;
+ Text lastRow = null;
+
+ boolean rowsKnown = false;
+
+ FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
+
+ if (mfi != null) {
+ firstRow = mfi.getFirstRow();
+ lastRow = mfi.getLastRow();
+ rowsKnown = true;
+ }
+
+ if (rowsKnown && firstRow.compareTo(midRow) > 0) {
+ // only in high
+ long highSize = entry.getValue().getSize();
+ long highEntries = entry.getValue().getNumEntries();
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
+ // only in low
+ long lowSize = entry.getValue().getSize();
+ long lowEntries = entry.getValue().getNumEntries();
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
+
+ highDatafilesToRemove.add(entry.getKey());
+ } else {
+ long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
+ long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
+ lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
+
+ long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
+ long highEntries = (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
+ highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
+ }
+ }
+ }
+
+ public static void deleteTable(String tableId, boolean insertDeletes, Credentials credentials, ZooLock lock) throws AccumuloException, IOException {
+ Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, Authorizations.EMPTY);
+ Text tableIdText = new Text(tableId);
+ BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000)
+ .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
+
+ // scan metadata for our table and delete everything we find
+ Mutation m = null;
+ ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
+
+ // insert deletes before deleting data from metadata... this makes the code fault tolerant
+ if (insertDeletes) {
+
+ ms.fetchColumnFamily(DataFileColumnFamily.NAME);
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms);
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
+
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ FileRef ref = new FileRef(VolumeManagerImpl.get(), key);
+ bw.addMutation(createDeleteMutation(tableId, ref.meta().toString()));
+ }
+
+ if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
+ bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
+ }
+ }
+
+ bw.flush();
+
+ ms.clearColumns();
+ }
+
+ for (Entry<Key,Value> cell : ms) {
+ Key key = cell.getKey();
+
+ if (m == null) {
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+
+ if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
+ bw.addMutation(m);
+ m = new Mutation(key.getRow());
+ if (lock != null)
+ putLockID(lock, m);
+ }
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ }
+
+ if (m != null)
+ bw.addMutation(m);
+
+ bw.close();
+ }
+
+ static String getZookeeperLogLocation() {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_WALOGS;
+ }
+
+ public static void addLogEntry(Credentials credentials, LogEntry entry, ZooLock zooLock) {
+ if (entry.extent.isRootTablet()) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID())) {
+ String[] parts = entry.filename.split("/");
+ String uniqueId = parts[parts.length - 1];
+ zoo.putPersistentData(root + "/" + uniqueId, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+ }
+ break;
+ } catch (KeeperException e) {
+ log.error(e, e);
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ } catch (IOException e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ } else {
+ Mutation m = new Mutation(entry.getRow());
+ m.put(entry.getColumnFamily(), entry.getColumnQualifier(), entry.getValue());
+ update(credentials, zooLock, m, entry.extent);
+ }
+ }
+
+ public static void setRootTabletDir(String dir) throws IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
+ try {
- zoo.putPersistentData(zpath, dir.getBytes(Constants.UTF8), -1, NodeExistsPolicy.OVERWRITE);
++ zoo.putPersistentData(zpath, dir.getBytes(UTF_8), -1, NodeExistsPolicy.OVERWRITE);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ }
+
+ public static String getRootTabletDir() throws IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String zpath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + RootTable.ZROOT_TABLET_PATH;
+ try {
- return new String(zoo.getData(zpath, null), Constants.UTF8);
++ return new String(zoo.getData(zpath, null), UTF_8);
+ } catch (KeeperException e) {
+ throw new IOException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ }
+
+ public static Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>> getFileAndLogEntries(Credentials credentials, KeyExtent extent) throws KeeperException,
+ InterruptedException, IOException {
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ TreeMap<FileRef,DataFileValue> sizes = new TreeMap<FileRef,DataFileValue>();
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ if (extent.isRootTablet()) {
+ getRootLogEntries(result);
+ Path rootDir = new Path(getRootTabletDir());
+ FileStatus[] files = fs.listStatus(rootDir);
+ for (FileStatus fileStatus : files) {
+ if (fileStatus.getPath().toString().endsWith("_tmp")) {
+ continue;
+ }
+ DataFileValue dfv = new DataFileValue(0, 0);
+ sizes.put(new FileRef(fileStatus.getPath().toString(), fileStatus.getPath()), dfv);
+ }
+
+ } else {
+ String systemTableToCheck = extent.isMeta() ? RootTable.ID : MetadataTable.ID;
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, systemTableToCheck, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.setRange(extent.toMetadataRange());
+
+ for (Entry<Key,Value> entry : scanner) {
+ if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
+ throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
+ }
+
+ if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
+ result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
+ } else if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ DataFileValue dfv = new DataFileValue(entry.getValue().get());
+ sizes.put(new FileRef(fs, entry.getKey()), dfv);
+ } else {
+ throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily());
+ }
+ }
+ }
+
+ return new Pair<List<LogEntry>,SortedMap<FileRef,DataFileValue>>(result, sizes);
+ }
+
+ public static List<LogEntry> getLogEntries(Credentials credentials, KeyExtent extent) throws IOException, KeeperException, InterruptedException {
+ log.info("Scanning logging entries for " + extent);
+ ArrayList<LogEntry> result = new ArrayList<LogEntry>();
+ if (extent.equals(RootTable.EXTENT)) {
+ log.info("Getting logs for root tablet from zookeeper");
+ getRootLogEntries(result);
+ } else {
+ log.info("Scanning metadata for logs used for tablet " + extent);
+ Scanner scanner = getTabletLogScanner(credentials, extent);
+ Text pattern = extent.getMetadataEntry();
+ for (Entry<Key,Value> entry : scanner) {
+ Text row = entry.getKey().getRow();
+ if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
+ if (row.equals(pattern)) {
+ result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ }
+
+ Collections.sort(result, new Comparator<LogEntry>() {
+ @Override
+ public int compare(LogEntry o1, LogEntry o2) {
+ long diff = o1.timestamp - o2.timestamp;
+ if (diff < 0)
+ return -1;
+ if (diff > 0)
+ return 1;
+ return 0;
+ }
+ });
+ log.info("Returning logs " + result + " for extent " + extent);
+ return result;
+ }
+
+ static void getRootLogEntries(ArrayList<LogEntry> result) throws KeeperException, InterruptedException, IOException {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ String root = getZookeeperLogLocation();
+ // there's a little race between getting the children and fetching
+ // the data. The log can be removed in between.
+ while (true) {
+ result.clear();
+ for (String child : zoo.getChildren(root)) {
+ LogEntry e = new LogEntry();
+ try {
+ e.fromBytes(zoo.getData(root + "/" + child, null));
+ // upgrade from !0;!0<< -> +r<<
+ e.extent = RootTable.EXTENT;
+ result.add(e);
+ } catch (KeeperException.NoNodeException ex) {
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private static Scanner getTabletLogScanner(Credentials credentials, KeyExtent extent) {
+ String tableId = MetadataTable.ID;
+ if (extent.isMeta())
+ tableId = RootTable.ID;
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, tableId, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ Text start = extent.getMetadataEntry();
+ Key endKey = new Key(start, LogColumnFamily.NAME);
+ endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
+ scanner.setRange(new Range(new Key(start), endKey));
+ return scanner;
+ }
+
+ private static class LogEntryIterator implements Iterator<LogEntry> {
+
+ Iterator<LogEntry> zookeeperEntries = null;
+ Iterator<LogEntry> rootTableEntries = null;
+ Iterator<Entry<Key,Value>> metadataEntries = null;
+
+ LogEntryIterator(Credentials creds) throws IOException, KeeperException, InterruptedException {
+ zookeeperEntries = getLogEntries(creds, RootTable.EXTENT).iterator();
+ rootTableEntries = getLogEntries(creds, new KeyExtent(new Text(MetadataTable.ID), null, null)).iterator();
+ try {
+ Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken())
+ .createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ log.info("Setting range to " + MetadataSchema.TabletsSection.getRange());
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ metadataEntries = scanner.iterator();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return zookeeperEntries.hasNext() || rootTableEntries.hasNext() || metadataEntries.hasNext();
+ }
+
+ @Override
+ public LogEntry next() {
+ if (zookeeperEntries.hasNext()) {
+ return zookeeperEntries.next();
+ }
+ if (rootTableEntries.hasNext()) {
+ return rootTableEntries.next();
+ }
+ Entry<Key,Value> entry = metadataEntries.next();
+ return LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static Iterator<LogEntry> getLogEntries(Credentials creds) throws IOException, KeeperException, InterruptedException {
+ return new LogEntryIterator(creds);
+ }
+
+ public static void removeUnusedWALEntries(KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
+ if (extent.isRootTablet()) {
+ for (LogEntry entry : logEntries) {
+ String root = getZookeeperLogLocation();
+ while (true) {
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.isLockHeld(zooLock.getLockID())) {
+ String parts[] = entry.filename.split("/");
+ zoo.recursiveDelete(root + "/" + parts[parts.length - 1], NodeMissingPolicy.SKIP);
+ }
+ break;
+ } catch (Exception e) {
+ log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+ }
+ }
+ } else {
+ Mutation m = new Mutation(extent.getMetadataEntry());
+ for (LogEntry entry : logEntries) {
+ m.putDelete(LogColumnFamily.NAME, new Text(entry.getName()));
+ }
+ update(SystemCredentials.get(), zooLock, m, extent);
+ }
+ }
+
+ private static void getFiles(Set<String> files, Map<Key,Value> tablet, String srcTableId) {
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (srcTableId != null && !cf.startsWith("../") && !cf.contains(":")) {
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ }
+ files.add(cf);
+ }
+ }
+ }
+
+ private static Mutation createCloneMutation(String srcTableId, String tableId, Map<Key,Value> tablet) {
+
+ KeyExtent ke = new KeyExtent(tablet.keySet().iterator().next().getRow(), (Text) null);
+ Mutation m = new Mutation(KeyExtent.getMetadataEntry(new Text(tableId), ke.getEndRow()));
+
+ for (Entry<Key,Value> entry : tablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ String cf = entry.getKey().getColumnQualifier().toString();
+ if (!cf.startsWith("../") && !cf.contains(":"))
+ cf = "../" + srcTableId + entry.getKey().getColumnQualifier();
+ m.put(entry.getKey().getColumnFamily(), new Text(cf), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
+ m.put(TabletsSection.LastLocationColumnFamily.NAME, entry.getKey().getColumnQualifier(), entry.getValue());
+ } else if (entry.getKey().getColumnFamily().equals(TabletsSection.LastLocationColumnFamily.NAME)) {
+ // skip
+ } else {
+ m.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getValue());
+ }
+ }
+ return m;
+ }
+
+ private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException {
+ String tableName = MetadataTable.NAME;
+ if (tableId.equals(MetadataTable.ID))
+ tableName = RootTable.NAME;
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ mscanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ mscanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
+ mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(mscanner);
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(mscanner);
+ return mscanner;
+ }
+
+ static void initializeClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator ti;
+ if (srcTableId.equals(MetadataTable.ID))
+ ti = new TabletIterator(createCloneScanner(srcTableId, conn), new Range(), true, true);
+ else
+ ti = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true, true);
+
+ if (!ti.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId);
+
+ while (ti.hasNext())
+ bw.addMutation(createCloneMutation(srcTableId, tableId, ti.next()));
+
+ bw.flush();
+ }
+
+ private static int compareEndRows(Text endRow1, Text endRow2) {
+ return new KeyExtent(new Text("0"), endRow1, null).compareTo(new KeyExtent(new Text("0"), endRow2, null));
+ }
+
+ static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw) throws TableNotFoundException, MutationsRejectedException {
+ TabletIterator srcIter = new TabletIterator(createCloneScanner(srcTableId, conn), new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(), true,
+ true);
+ TabletIterator cloneIter = new TabletIterator(createCloneScanner(tableId, conn), new KeyExtent(new Text(tableId), null, null).toMetadataRange(), true, true);
+
+ if (!cloneIter.hasNext() || !srcIter.hasNext())
+ throw new RuntimeException(" table deleted during clone? srcTableId = " + srcTableId + " tableId=" + tableId);
+
+ int rewrites = 0;
+
+ while (cloneIter.hasNext()) {
+ Map<Key,Value> cloneTablet = cloneIter.next();
+ Text cloneEndRow = new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ HashSet<String> cloneFiles = new HashSet<String>();
+
+ boolean cloneSuccessful = false;
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ if (entry.getKey().getColumnFamily().equals(ClonedColumnFamily.NAME)) {
+ cloneSuccessful = true;
+ break;
+ }
+ }
+
+ if (!cloneSuccessful)
+ getFiles(cloneFiles, cloneTablet, null);
+
+ List<Map<Key,Value>> srcTablets = new ArrayList<Map<Key,Value>>();
+ Map<Key,Value> srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+
+ Text srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+
+ int cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
+
+ HashSet<String> srcFiles = new HashSet<String>();
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+
+ while (cmp > 0) {
+ srcTablet = srcIter.next();
+ srcTablets.add(srcTablet);
+ srcEndRow = new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
+ cmp = compareEndRows(cloneEndRow, srcEndRow);
+ if (cmp < 0)
+ throw new TabletIterator.TabletDeletedException("Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
+
+ if (!cloneSuccessful)
+ getFiles(srcFiles, srcTablet, srcTableId);
+ }
+
+ if (cloneSuccessful)
+ continue;
+
+ if (!srcFiles.containsAll(cloneFiles)) {
+ // delete existing cloned tablet entry
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
+
+ for (Entry<Key,Value> entry : cloneTablet.entrySet()) {
+ Key k = entry.getKey();
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
+ }
+
+ bw.addMutation(m);
+
+ for (Map<Key,Value> st : srcTablets)
+ bw.addMutation(createCloneMutation(srcTableId, tableId, st));
+
+ rewrites++;
+ } else {
+ // write out marker that this tablet was successfully cloned
+ Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
- m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(Constants.UTF8)));
++ m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+ }
+
+ bw.flush();
+ return rewrites;
+ }
+
+ public static void cloneTable(Instance instance, String srcTableId, String tableId, VolumeManager volumeManager) throws Exception {
+
+ Connector conn = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ while (true) {
+
+ try {
+ initializeClone(srcTableId, tableId, conn, bw);
+
+ // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed
+
+ while (true) {
+ int rewrites = checkClone(srcTableId, tableId, conn, bw);
+
+ if (rewrites == 0)
+ break;
+ }
+
+ bw.flush();
+ break;
+
+ } catch (TabletIterator.TabletDeletedException tde) {
+ // tablets were merged in the src table
+ bw.flush();
+
+ // delete what we have cloned and try again
+ deleteTable(tableId, false, SystemCredentials.get(), null);
+
+ log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
+
+ UtilWaitThread.sleep(100);
+ }
+ }
+
+ // delete the clone markers and create directory entries
+ Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
+
+ int dirCount = 0;
+
+ for (Entry<Key,Value> entry : mscanner) {
+ Key k = entry.getKey();
+ Mutation m = new Mutation(k.getRow());
+ m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
+ String dir = volumeManager.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId
+ + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES));
- TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(Constants.UTF8)));
++ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ }
+
+ public static void chopped(KeyExtent extent, ZooLock zooLock) {
+ Mutation m = new Mutation(extent.getMetadataEntry());
- ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(Constants.UTF8)));
++ ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(UTF_8)));
+ update(SystemCredentials.get(), zooLock, m, extent);
+ }
+
+ public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+ mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+ mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ for (Entry<Key,Value> entry : mscanner) {
+ log.debug("Looking at entry " + entry + " with tid " + tid);
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ log.debug("deleting entry " + entry);
+ Mutation m = new Mutation(entry.getKey().getRow());
+ m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ bw.addMutation(m);
+ }
+ }
+ bw.close();
+ }
+
+ public static List<FileRef> getBulkFilesLoaded(Connector conn, KeyExtent extent, long tid) throws IOException {
+ List<FileRef> result = new ArrayList<FileRef>();
+ try {
+ VolumeManager fs = VolumeManagerImpl.get();
+ Scanner mscanner = new IsolatedScanner(conn.createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY));
+ mscanner.setRange(extent.toMetadataRange());
+ mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : mscanner) {
+ if (Long.parseLong(entry.getValue().toString()) == tid) {
+ result.add(new FileRef(fs, entry.getKey()));
+ }
+ }
+ return result;
+ } catch (TableNotFoundException ex) {
+ // unlikely
+ throw new RuntimeException("Onos! teh metadata table has vanished!!");
+ }
+ }
+
+ public static Map<FileRef,Long> getBulkFilesLoaded(Credentials credentials, KeyExtent extent) throws IOException {
+ Text metadataRow = extent.getMetadataEntry();
+ Map<FileRef,Long> ret = new HashMap<FileRef,Long>();
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, extent.isMeta() ? RootTable.ID : MetadataTable.ID, Authorizations.EMPTY);
+ scanner.setRange(new Range(metadataRow));
+ scanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : scanner) {
+ Long tid = Long.parseLong(entry.getValue().toString());
+ ret.put(new FileRef(fs, entry.getKey()), tid);
+ }
+ return ret;
+ }
+
+ public static void addBulkLoadInProgressFlag(String path) {
+
+ Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
+ m.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
+
+ // new KeyExtent is only added to force update to write to the metadata table, not the root table
+ // because bulk loads aren't supported to the metadata table
+ update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
+ }
+
+ public static void removeBulkLoadInProgressFlag(String path) {
+
+ Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+
+ // new KeyExtent is only added to force update to write to the metadata table, not the root table
+ // because bulk loads aren't supported to the metadata table
+ update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
+ }
+
+ /**
+ * During an upgrade we need to move deletion requests for files under the !METADATA table to the root tablet.
+ */
+ public static void moveMetaDeleteMarkers(Instance instance, Credentials creds) {
+ String oldDeletesPrefix = "!!~del";
+ Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false);
+
+ // move old delete markers to new location, to standardize table schema between all metadata tables
+ Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY);
+ scanner.setRange(oldDeletesRange);
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(oldDeletesPrefix)) {
+ moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row, oldDeletesPrefix);
+ } else {
+ break;
+ }
+ }
+
+ }
+
+ public static void moveMetaDeleteMarkersFrom14(Instance instance, Credentials creds) {
+ // new KeyExtent is only added to force update to write to the metadata table, not the root table
+ KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"), null, null);
+
+ // move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table
+ Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID, Authorizations.EMPTY);
+ scanner.setRange(MetadataSchema.DeletesSection.getRange());
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" + MetadataTable.ID)) {
+ moveDeleteEntry(creds, notMetadata, entry, row, MetadataSchema.DeletesSection.getRowPrefix());
+ } else {
+ break;
+ }
+ }
+ }
+
+ private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent, Entry<Key,Value> entry, String rowID, String prefix) {
+ String filename = rowID.substring(prefix.length());
+
+ // add the new entry first
+ log.info("Moving " + filename + " marker in " + RootTable.NAME);
+ Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename);
+ m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
+ update(creds, m, RootTable.EXTENT);
+
+ // then remove the old entry
+ m = new Mutation(entry.getKey().getRow());
+ m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
+ update(creds, m, oldExtent);
+ }
+
+ public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
+ TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
+
+ HashSet<ColumnFQ> colSet = null;
+ if (columns != null) {
+ colSet = new HashSet<ColumnFQ>(columns);
+ }
+
+ for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
+
+ if (columns != null && !colSet.contains(new ColumnFQ(entry.getKey()))) {
+ continue;
+ }
+
+ Text row = entry.getKey().getRow();
+
+ SortedMap<ColumnFQ,Value> colVals = tabletEntries.get(row);
+ if (colVals == null) {
+ colVals = new TreeMap<ColumnFQ,Value>();
+ tabletEntries.put(row, colVals);
+ }
+
+ colVals.put(new ColumnFQ(entry.getKey()), entry.getValue());
+ }
+
+ return tabletEntries;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index 4e5df9e,0000000..c0d6894
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+public class NamespacePropUtil {
+ public static boolean setNamespaceProperty(String namespaceId, String property, String value) throws KeeperException, InterruptedException {
+ if (!isPropertyValid(property, value))
+ return false;
+
+ // create the zk node for per-namespace properties for this namespace if it doesn't already exist
+ String zkNamespacePath = getPath(namespaceId);
+ ZooReaderWriter.getInstance().putPersistentData(zkNamespacePath, new byte[0], NodeExistsPolicy.SKIP);
+
+ // create the zk node for this property and set it's data to the specified value
+ String zPath = zkNamespacePath + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ return true;
+ }
+
+ public static boolean isPropertyValid(String property, String value) {
+ Property p = Property.getPropertyByKey(property);
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
+ return false;
+
+ return true;
+ }
+
+ public static void removeNamespaceProperty(String namespaceId, String property) throws InterruptedException, KeeperException {
+ String zPath = getPath(namespaceId) + "/" + property;
+ ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
+ }
+
+ private static String getPath(String namespaceId) {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZNAMESPACES + "/" + namespaceId + Constants.ZNAMESPACE_CONF;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
index dd540f2,0000000..8924c85
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RandomizeVolumes.java
@@@ -1,131 -1,0 +1,131 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.util.Map.Entry;
+
- import org.apache.accumulo.core.Constants;
-
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.tables.TableManager;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+
+public class RandomizeVolumes {
+ private static final Logger log = Logger.getLogger(RandomizeVolumes.class);
+
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException {
+ ClientOnRequiredTable opts = new ClientOnRequiredTable();
+ opts.parseArgs(RandomizeVolumes.class.getName(), args);
+ Connector c;
+ if (opts.getToken() == null) {
+ SystemCredentials creds = SystemCredentials.get();
+ c = opts.getInstance().getConnector(creds.getPrincipal(), creds.getToken());
+ } else {
+ c = opts.getConnector();
+ }
+ try {
+ int status = randomize(c, opts.tableName);
+ System.exit(status);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ System.exit(4);
+ }
+ }
+
+ public static int randomize(Connector c, String tableName) throws IOException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ VolumeManager vm = VolumeManagerImpl.get();
+ if (vm.getVolumes().size() < 2) {
+ log.error("There are not enough volumes configured");
+ return 1;
+ }
+ String tableId = c.tableOperations().tableIdMap().get(tableName);
+ if (null == tableId) {
+ log.error("Could not determine the table ID for table " + tableName);
+ return 2;
+ }
+ TableState tableState = TableManager.getInstance().getTableState(tableId);
+ if (TableState.OFFLINE != tableState) {
+ log.info("Taking " + tableName + " offline");
+ c.tableOperations().offline(tableName, true);
+ log.info(tableName + " offline");
+ }
+ log.info("Rewriting entries for " + tableName);
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ DIRECTORY_COLUMN.fetch(scanner);
+ scanner.setRange(TabletsSection.getRange(tableId));
+ BatchWriter writer = c.createBatchWriter(MetadataTable.NAME, null);
+ int count = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ String oldLocation = entry.getValue().toString();
+ String directory;
+ if (oldLocation.contains(":")) {
+ String[] parts = oldLocation.split(Path.SEPARATOR);
+ String tableIdEntry = parts[parts.length - 2];
+ if (!tableIdEntry.equals(tableId)) {
+ log.error("Unexpected table id found: " + tableIdEntry + ", expected " + tableId + "; skipping");
+ continue;
+ }
+ directory = parts[parts.length - 1];
+ } else {
+ directory = oldLocation.substring(Path.SEPARATOR.length());
+ }
+ Key key = entry.getKey();
+ Mutation m = new Mutation(key.getRow());
+
+ String newLocation = vm.choose(ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + directory;
- m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(Constants.UTF8)));
++ m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newLocation.getBytes(UTF_8)));
+ if (log.isTraceEnabled()) {
+ log.trace("Replacing " + oldLocation + " with " + newLocation);
+ }
+ writer.addMutation(m);
+ try {
+ vm.mkdirs(new Path(newLocation));
+ } catch (IOException ex) {
+ // nevermind
+ }
+ count++;
+ }
+ writer.close();
+ log.info("Updated " + count + " entries for table " + tableName);
+ if (TableState.OFFLINE != tableState) {
+ c.tableOperations().online(tableName, true);
+ log.info("table " + tableName + " back online");
+ }
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index a08000e,0000000..a82fa14
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@@ -1,124 -1,0 +1,125 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Stack;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.beust.jcommander.Parameter;
+
+public class RestoreZookeeper {
+
+ private static class Restore extends DefaultHandler {
+ IZooReaderWriter zk = null;
+ Stack<String> cwd = new Stack<String>();
+ boolean overwrite = false;
+
+ Restore(IZooReaderWriter zk, boolean overwrite) {
+ this.zk = zk;
+ this.overwrite = overwrite;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
+ if ("node".equals(name)) {
+ String child = attributes.getValue("name");
+ if (child == null)
+ throw new RuntimeException("name attribute not set");
+ String encoding = attributes.getValue("encoding");
+ String value = attributes.getValue("value");
+ if (value == null)
+ value = "";
+ String path = cwd.lastElement() + "/" + child;
+ create(path, value, encoding);
+ cwd.push(path);
+ } else if ("dump".equals(name)) {
+ String root = attributes.getValue("root");
+ if (root.equals("/"))
+ cwd.push("");
+ else
+ cwd.push(root);
- create(root, "", Constants.UTF8.name());
++ create(root, "", UTF_8.name());
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws SAXException {
+ cwd.pop();
+ }
+
+ // assume UTF-8 if not "base64"
+ private void create(String path, String value, String encoding) {
- byte[] data = value.getBytes(Constants.UTF8);
++ byte[] data = value.getBytes(UTF_8);
+ if ("base64".equals(encoding))
+ data = Base64.decodeBase64(data);
+ try {
+ try {
+ zk.putPersistentData(path, data, overwrite ? NodeExistsPolicy.OVERWRITE : NodeExistsPolicy.FAIL);
+ } catch (KeeperException e) {
+ if (e.code().equals(KeeperException.Code.NODEEXISTS))
+ throw new RuntimeException(path + " exists. Remove it first.");
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class Opts extends Help {
+ @Parameter(names = {"-z", "--keepers"})
+ String keepers = "localhost:2181";
+ @Parameter(names = "--overwrite")
+ boolean overwrite = false;
+ @Parameter(names = "--file")
+ String file;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Logger.getRootLogger().setLevel(Level.WARN);
+ Opts opts = new Opts();
+ opts.parseArgs(RestoreZookeeper.class.getName(), args);
+
+ InputStream in = System.in;
+ if (opts.file != null) {
+ in = new FileInputStream(opts.file);
+ }
+
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser parser = factory.newSAXParser();
+ parser.parse(in, new Restore(ZooReaderWriter.getInstance(), opts.overwrite));
+ in.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index 85f29ba,0000000..b1a553d
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@@ -1,277 -1,0 +1,278 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.Socket;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.SocketFactory;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang.math.LongRange;
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.Filter;
+import org.apache.log4j.spi.LocationInfo;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+import org.apache.log4j.varia.LevelRangeFilter;
+import org.apache.log4j.xml.XMLLayout;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+
+public class SendLogToChainsaw extends XMLLayout {
+
+ private static Pattern logPattern = Pattern.compile(
+ "^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
+
+ private File[] logFiles = null;
+
+ private SocketFactory factory = SocketFactory.getDefault();
+
+ private WildcardFileFilter fileFilter = null;
+
+ private Socket socket = null;
+
+ private Pattern lineFilter = null;
+
+ private LongRange dateFilter = null;
+
+ private LevelRangeFilter levelFilter = null;
+
+ public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
+
+ // Set up the file name filter
+ if (null != fileNameFilter) {
+ fileFilter = new WildcardFileFilter(fileNameFilter);
+ } else {
+ fileFilter = new WildcardFileFilter("*");
+ }
+
+ // Get the list of files that match
+ File dir = new File(directory);
+ if (dir.isDirectory()) {
+ logFiles = dir.listFiles((FilenameFilter) fileFilter);
+ } else {
+ throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
+ }
+
+ if (logFiles.length == 0) {
+ throw new IllegalArgumentException("No files match the supplied filter.");
+ }
+
+ socket = factory.createSocket(host, port);
+
+ lineFilter = Pattern.compile(regex);
+
+ // Create Date Filter
+ if (null != start) {
+ if (end == null)
+ end = new Date(System.currentTimeMillis());
+ dateFilter = new LongRange(start.getTime(), end.getTime());
+ }
+
+ if (null != level) {
+ Level base = Level.toLevel(level.toUpperCase());
+ levelFilter = new LevelRangeFilter();
+ levelFilter.setAcceptOnMatch(true);
+ levelFilter.setLevelMin(base);
+ levelFilter.setLevelMax(Level.FATAL);
+ }
+ }
+
+ public void processLogFiles() throws IOException {
+ String line = null;
+ String out = null;
+ InputStreamReader isReader = null;
+ BufferedReader reader = null;
+ try {
+ for (File log : logFiles) {
+ // Parse the server type and name from the log file name
+ String threadName = log.getName().substring(0, log.getName().indexOf("."));
+ try {
- isReader = new InputStreamReader(new FileInputStream(log), Constants.UTF8);
++ isReader = new InputStreamReader(new FileInputStream(log), UTF_8);
+ } catch (FileNotFoundException e) {
+ System.out.println("Unable to find file: " + log.getAbsolutePath());
+ throw e;
+ }
+ reader = new BufferedReader(isReader);
+
+ try {
+ line = reader.readLine();
+ while (null != line) {
+ out = convertLine(line, threadName);
+ if (null != out) {
+ if (socket != null && socket.isConnected())
- socket.getOutputStream().write(out.getBytes(Constants.UTF8));
++ socket.getOutputStream().write(out.getBytes(UTF_8));
+ else
+ System.err.println("Unable to send data to transport");
+ }
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ System.out.println("Error processing line: " + line + ". Output was " + out);
+ throw e;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ if (isReader != null) {
+ isReader.close();
+ }
+ }
+ }
+ } finally {
+ if (socket != null && socket.isConnected()) {
+ socket.close();
+ }
+ }
+ }
+
+ private String convertLine(String line, String threadName) throws UnsupportedEncodingException {
+ String result = null;
+ Matcher m = logPattern.matcher(line);
+ if (m.matches()) {
+
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(new Date(System.currentTimeMillis()));
+ Integer date = Integer.parseInt(m.group(1));
+ Integer hour = Integer.parseInt(m.group(2));
+ Integer min = Integer.parseInt(m.group(3));
+ Integer sec = Integer.parseInt(m.group(4));
+ Integer ms = Integer.parseInt(m.group(5));
+ String clazz = m.group(6);
+ String level = m.group(7);
+ String message = m.group(8);
+ // Apply the regex filter if supplied
+ if (null != lineFilter) {
+ Matcher match = lineFilter.matcher(message);
+ if (!match.matches())
+ return null;
+ }
+ // URL encode the message
- message = URLEncoder.encode(message, Constants.UTF8.name());
++ message = URLEncoder.encode(message, UTF_8.name());
+ // Assume that we are processing logs from today.
+ // If the date in the line is greater than today, then it must be
+ // from the previous month.
+ cal.set(Calendar.DATE, date);
+ cal.set(Calendar.HOUR_OF_DAY, hour);
+ cal.set(Calendar.MINUTE, min);
+ cal.set(Calendar.SECOND, sec);
+ cal.set(Calendar.MILLISECOND, ms);
+ if (date > cal.get(Calendar.DAY_OF_MONTH)) {
+ cal.add(Calendar.MONTH, -1);
+ }
+ long ts = cal.getTimeInMillis();
+ // If this event is not between the start and end dates, then skip it.
+ if (null != dateFilter && !dateFilter.containsLong(ts))
+ return null;
+ Category c = Logger.getLogger(clazz);
+ Level l = Level.toLevel(level);
+ LoggingEvent event = new LoggingEvent(clazz, c, ts, l, message, threadName, (ThrowableInformation) null, (String) null, (LocationInfo) null,
+ (Map<?,?>) null);
+ // Check the log level filter
+ if (null != levelFilter && (levelFilter.decide(event) == Filter.DENY)) {
+ return null;
+ }
+ result = format(event);
+ }
+ return result;
+ }
+
+ private static class DateConverter implements IStringConverter<Date> {
+ @Override
+ public Date convert(String value) {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+ try {
+ return formatter.parse(value);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private static class Opts extends Help {
+
+ @Parameter(names = {"-d", "--logDirectory"}, description = "ACCUMULO log directory path", required = true)
+ String dir;
+
+ @Parameter(names = {"-f", "--fileFilter"}, description = "filter to apply to names of logs")
+ String filter;
+
+ @Parameter(names = {"-h", "--host"}, description = "host where chainsaw is running", required = true)
+ String hostname;
+
+ @Parameter(names = {"-p", "--port"}, description = "port where XMLSocketReceiver is listening", required = true)
+ int portnum;
+
+ @Parameter(names = {"-s", "--start"}, description = "start date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
+ Date startDate;
+
+ @Parameter(names = {"-e", "--end"}, description = "end date filter (yyyyMMddHHmmss)", required = true, converter = DateConverter.class)
+ Date endDate;
+
+ @Parameter(names = {"-l", "--level"}, description = "filter log level")
+ String level;
+
+ @Parameter(names = {"-m", "--messageFilter"}, description = "regex filter for log messages")
+ String regex;
+ }
+
+ /**
+ *
+ * @param args
+ * <ol>
+ * <li>path to log directory</li>
+ * <li>filter to apply for logs to include (uses wildcards (i.e. logger* and IS case sensitive)</li>
+ * <li>chainsaw host</li>
+ * <li>chainsaw port</li>
+ * <li>start date filter</li>
+ * <li>end date filter</li>
+ * <li>optional regex filter to match on each log4j message</li>
+ * <li>optional level filter</li>
+ * </ol>
+ */
+ public static void main(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(SendLogToChainsaw.class.getName(), args);
+
+ SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
+ c.processLogFiles();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index b6ca527,0000000..401d98c
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@@ -1,45 -1,0 +1,47 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+public class SystemPropUtil {
+ public static boolean setSystemProperty(String property, String value) throws KeeperException, InterruptedException {
+ Property p = Property.getPropertyByKey(property);
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property))
+ return false;
+
+ // create the zk node for this property and set it's data to the specified value
+ String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ return true;
+ }
+
+ public static void removeSystemProperty(String property) throws InterruptedException, KeeperException {
+ String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
+ ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.FAIL);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index bcaf9b0,0000000..6ee1593
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@@ -1,60 -1,0 +1,62 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+public class TablePropUtil {
+ public static boolean setTableProperty(String tableId, String property, String value) throws KeeperException, InterruptedException {
+ if (!isPropertyValid(property, value))
+ return false;
+
+ // create the zk node for per-table properties for this table if it doesn't already exist
+ String zkTablePath = getTablePath(tableId);
+ ZooReaderWriter.getInstance().putPersistentData(zkTablePath, new byte[0], NodeExistsPolicy.SKIP);
+
+ // create the zk node for this property and set it's data to the specified value
+ String zPath = zkTablePath + "/" + property;
- ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+
+ return true;
+ }
+
+ public static boolean isPropertyValid(String property, String value) {
+ Property p = Property.getPropertyByKey(property);
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidTablePropertyKey(property))
+ return false;
+
+ return true;
+ }
+
+ public static void removeTableProperty(String tableId, String property) throws InterruptedException, KeeperException {
+ String zPath = getTablePath(tableId) + "/" + property;
+ ZooReaderWriter.getInstance().recursiveDelete(zPath, NodeMissingPolicy.SKIP);
+ }
+
+ private static String getTablePath(String tablename) {
+ return ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" + tablename + Constants.ZTABLE_CONF;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index 34c2151,0000000..edbb103
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@@ -1,72 -1,0 +1,74 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.List;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+
+import com.beust.jcommander.Parameter;
+
+public class TabletServerLocks {
+
+ static class Opts extends Help {
+ @Parameter(names="-list")
+ boolean list = false;
+ @Parameter(names="-delete")
+ String delete = null;
+ }
+ public static void main(String[] args) throws Exception {
+
+ Instance instance = HdfsZooInstance.getInstance();
+ String tserverPath = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ Opts opts = new Opts();
+ opts.parseArgs(TabletServerLocks.class.getName(), args);
+
+ ZooCache cache = new ZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+
+ if (opts.list) {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ List<String> tabletServers = zoo.getChildren(tserverPath);
+
+ for (String tabletServer : tabletServers) {
+ byte[] lockData = ZooLock.getLockData(cache, tserverPath + "/" + tabletServer, null);
+ String holder = null;
+ if (lockData != null) {
- holder = new String(lockData, Constants.UTF8);
++ holder = new String(lockData, UTF_8);
+ }
+
+ System.out.printf("%32s %16s%n", tabletServer, holder);
+ }
+ } else if (opts.delete != null) {
+ ZooLock.deleteLock(tserverPath + "/" + args[1]);
+ } else {
+ System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");
+ }
+
+ }
+
+}
[48/50] [abbrv] Merge branch '1.6'
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
index 339c9a8,0000000..90a8622
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@@ -1,796 -1,0 +1,796 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
+import org.apache.accumulo.core.client.mock.impl.MockTabletLocator;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @since 1.6.0
+ */
+public class InputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Scanner}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ScanOpts {
+ TABLE_NAME, AUTHORIZATIONS, RANGES, COLUMNS, ITERATORS, TABLE_CONFIGS
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ AUTO_ADJUST_RANGES, SCAN_ISOLATION, USE_LOCAL_ITERATORS, SCAN_OFFLINE
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
+ checkArgument(tableName != null, "tableName is null");
+ conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @since 1.6.0
+ */
+ public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param auths
+ * the user's authorizations
+ * @since 1.6.0
+ */
+ public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf, Authorizations auths) {
+ if (auths != null && !auths.isEmpty())
+ conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the Accumulo scan authorizations
+ * @since 1.6.0
+ * @see #setScanAuthorizations(Class, Configuration, Authorizations)
+ */
+ public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration conf) {
+ String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
- return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(StandardCharsets.UTF_8));
++ return authString == null ? Authorizations.EMPTY : new Authorizations(authString.getBytes(UTF_8));
+ }
+
+ /**
+ * Sets the input ranges to scan on all input tables for this job. If not set, the entire table will be scanned.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param ranges
+ * the ranges that will be mapped over
+ * @throws IllegalArgumentException
+ * if the ranges cannot be encoded into base 64
+ * @since 1.6.0
+ */
+ public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
+ checkArgument(ranges != null, "ranges is null");
+
+ ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
+ try {
+ for (Range r : ranges) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ r.write(new DataOutputStream(baos));
+ rangeStrings.add(Base64.encodeBase64String(baos.toByteArray()));
+ }
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES), rangeStrings.toArray(new String[0]));
+ } catch (IOException ex) {
+ throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
+ }
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.6.0
+ * @see #setRanges(Class, Configuration, Collection)
+ */
+ public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
+
+ Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
+ List<Range> ranges = new ArrayList<Range>();
+ for (String rangeString : encodedRanges) {
- ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes(StandardCharsets.UTF_8)));
++ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes(UTF_8)));
+ Range range = new Range();
+ range.readFields(new DataInputStream(bais));
+ ranges.add(range);
+ }
+ return ranges;
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a list of iterators
+ * @since 1.6.0
+ * @see #addIterator(Class, Configuration, IteratorSetting)
+ */
+ public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
+ String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
+
+ // If no iterators are present, return an empty list
+ if (iterators == null || iterators.isEmpty())
+ return new ArrayList<IteratorSetting>();
+
+ // Compose the set of iterators encoded in the job configuration
+ StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
+ List<IteratorSetting> list = new ArrayList<IteratorSetting>();
+ try {
+ while (tokens.hasMoreTokens()) {
+ String itstring = tokens.nextToken();
- ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(StandardCharsets.UTF_8)));
++ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(UTF_8)));
+ list.add(new IteratorSetting(new DataInputStream(bais)));
+ bais.close();
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("couldn't decode iterator settings");
+ }
+ return list;
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for the single input table on this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If the column qualifier is null, the entire column family is
+ * selected. An empty set is the default and is equivalent to scanning the all columns.
+ * @throws IllegalArgumentException
+ * if the column family is null
+ * @since 1.6.0
+ */
+ public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ checkArgument(columnFamilyColumnQualifierPairs != null, "columnFamilyColumnQualifierPairs is null");
+ String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
+ }
+
+ public static String[] serializeColumns(Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ checkArgument(columnFamilyColumnQualifierPairs != null, "columnFamilyColumnQualifierPairs is null");
+ ArrayList<String> columnStrings = new ArrayList<String>(columnFamilyColumnQualifierPairs.size());
+ for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
+
+ if (column.getFirst() == null)
+ throw new IllegalArgumentException("Column family can not be null");
+
+ String col = Base64.encodeBase64String(TextUtil.getBytes(column.getFirst()));
+ if (column.getSecond() != null)
+ col += ":" + Base64.encodeBase64String(TextUtil.getBytes(column.getSecond()));
+ columnStrings.add(col);
+ }
+
+ return columnStrings.toArray(new String[0]);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a set of columns
+ * @since 1.6.0
+ * @see #fetchColumns(Class, Configuration, Collection)
+ */
+ public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass, Configuration conf) {
+ checkArgument(conf != null, "conf is null");
+ String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
+ List<String> serialized = new ArrayList<String>();
+ if (confValue != null) {
+ // Split and include any trailing empty strings to allow empty column families
+ for (String val : confValue.split(",", -1)) {
+ serialized.add(val);
+ }
+ }
+ return deserializeFetchedColumns(serialized);
+ }
+
+ public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
+ Set<Pair<Text,Text>> columns = new HashSet<Pair<Text,Text>>();
+
+ if (null == serialized) {
+ return columns;
+ }
+
+ for (String col : serialized) {
+ int idx = col.indexOf(":");
- Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(StandardCharsets.UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(
- StandardCharsets.UTF_8)));
- Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(StandardCharsets.UTF_8)));
++ Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(UTF_8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(
++ UTF_8)));
++ Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(UTF_8)));
+ columns.add(new Pair<Text,Text>(cf, cq));
+ }
+ return columns;
+ }
+
+ /**
+ * Encode an iterator on the input for the single input table associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param cfg
+ * the configuration of the iterator
+ * @throws IllegalArgumentException
+ * if the iterator can't be serialized into the configuration
+ * @since 1.6.0
+ */
+ public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String newIter;
+ try {
+ cfg.write(new DataOutputStream(baos));
+ newIter = Base64.encodeBase64String(baos.toByteArray());
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize IteratorSetting");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
+ String iterators = conf.get(confKey);
+ // No iterators specified yet, create a new string
+ if (iterators == null || iterators.isEmpty()) {
+ iterators = newIter;
+ } else {
+ // append the next iterator & reset
+ iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
+ }
+ // Store the iterators w/ the job
+ conf.set(confKey, iterators);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping ranges, then splits them to align with tablet boundaries.
+ * Disabling this feature will cause exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Class, Configuration, Collection)
+ * @since 1.6.0
+ */
+ public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(Class, Configuration, boolean)
+ */
+ public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setScanIsolation(Class, Configuration, boolean)
+ */
+ public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setLocalIterators(Class, Configuration, boolean)
+ */
+ public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
+ }
+
+ /**
+ * <p>
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are scanned. This will make the map reduce job directly read the
+ * table's files. If the table is not offline, then the job will fail. If the table comes online during the map reduce job, it is likely that the job will
+ * fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any iterators that are configured for the table will need to be
+ * on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as the input table for a map reduce job. If you plan to map
+ * reduce over the data many times, it may be better to the compact the table, clone it, take it offline, and use the clone for all map reduce jobs. The
+ * reason to do this is that compaction will reduce each tablet in the table to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may see better read performance. Second, it will support
+ * speculative execution better. When reading an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineTableScan(Class, Configuration, boolean)
+ */
+ public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
+ }
+
+ /**
+ * Sets configurations for multiple tables at a time.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param configs
+ * an array of {@link InputTableConfig} objects to associate with the job
+ * @since 1.6.0
+ */
+ public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf, Map<String,InputTableConfig> configs) {
+ MapWritable mapWritable = new MapWritable();
+ for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
+ mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ mapWritable.write(new DataOutputStream(baos));
+ } catch (IOException e) {
+ throw new IllegalStateException("Table configuration could not be serialized.");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
+ conf.set(confKey, Base64.encodeBase64String(baos.toByteArray()));
+ }
+
+ /**
+ * Returns all {@link InputTableConfig} objects associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return all of the table query configs for the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass, Configuration conf) {
+ Map<String,InputTableConfig> configs = new HashMap<String,InputTableConfig>();
+ Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass, conf);
+ if (defaultConfig != null)
+ configs.put(defaultConfig.getKey(), defaultConfig.getValue());
+ String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
+ MapWritable mapWritable = new MapWritable();
+ if (configString != null) {
+ try {
- byte[] bytes = Base64.decodeBase64(configString.getBytes(StandardCharsets.UTF_8));
++ byte[] bytes = Base64.decodeBase64(configString.getBytes(UTF_8));
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ mapWritable.readFields(new DataInputStream(bais));
+ bais.close();
+ } catch (IOException e) {
+ throw new IllegalStateException("The table query configurations could not be deserialized from the given configuration");
+ }
+ }
+ for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
+ configs.put(((Text) entry.getKey()).toString(), (InputTableConfig) entry.getValue());
+
+ return configs;
+ }
+
+ /**
+ * Returns the {@link InputTableConfig} for the given table
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table name for which to fetch the table query config
+ * @return the table query config for the given table name (if it exists) and null if it does not
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf, String tableName) {
+ Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
+ return queryConfigs.get(tableName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableId
+ * The table id for which to initialize the {@link TabletLocator}
+ * @return an Accumulo tablet locator
+ * @throws TableNotFoundException
+ * if the table name set on the configuration doesn't exist
+ * @since 1.6.0
+ */
+ public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf, String tableId) throws TableNotFoundException {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
+ if ("MockInstance".equals(instanceType))
+ return new MockTabletLocator();
+ Instance instance = getInstance(implementingClass, conf);
+ return TabletLocator.getLocator(instance, new Text(tableId));
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @throws IOException
+ * if the context is improperly configured
+ * @since 1.6.0
+ */
+ public static void validateOptions(Class<?> implementingClass, Configuration conf) throws IOException {
+
+ Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
+ if (!isConnectorInfoSet(implementingClass, conf))
+ throw new IOException("Input info has not been set.");
+ String instanceKey = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE));
+ if (!"MockInstance".equals(instanceKey) && !"ZooKeeperInstance".equals(instanceKey))
+ throw new IOException("Instance info has not been set.");
+ // validate that we can connect as configured
+ try {
+ String principal = getPrincipal(implementingClass, conf);
+ AuthenticationToken token = getAuthenticationToken(implementingClass, conf);
+ Connector c = getInstance(implementingClass, conf).getConnector(principal, token);
+ if (!c.securityOperations().authenticateUser(principal, token))
+ throw new IOException("Unable to authenticate user");
+
+ if (getInputTableConfigs(implementingClass, conf).size() == 0)
+ throw new IOException("No table set.");
+
+ for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
+ if (!c.securityOperations().hasTablePermission(getPrincipal(implementingClass, conf), tableConfig.getKey(), TablePermission.READ))
+ throw new IOException("Unable to access table");
+ }
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+ if (!tableConfig.shouldUseLocalIterators()) {
+ if (tableConfig.getIterators() != null) {
+ for (IteratorSetting iter : tableConfig.getIterators()) {
+ if (!c.tableOperations().testClassLoad(tableConfigEntry.getKey(), iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
+ throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
+ }
+ }
+ }
+ }
+ } catch (AccumuloException e) {
+ throw new IOException(e);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the configuration based on the properties set using the single-table
+ * input methods.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop instance for which to retrieve the configuration
+ * @return the config object built from the single input table properties set on the job
+ * @since 1.6.0
+ */
+ protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(Class<?> implementingClass, Configuration conf) {
+ String tableName = getInputTableName(implementingClass, conf);
+ if (tableName != null) {
+ InputTableConfig queryConfig = new InputTableConfig();
+ List<IteratorSetting> itrs = getIterators(implementingClass, conf);
+ if (itrs != null)
+ queryConfig.setIterators(itrs);
+ Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
+ if (columns != null)
+ queryConfig.fetchColumns(columns);
+ List<Range> ranges = null;
+ try {
+ ranges = getRanges(implementingClass, conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (ranges != null)
+ queryConfig.setRanges(ranges);
+
+ queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf)).setUseIsolatedScanners(isIsolated(implementingClass, conf))
+ .setUseLocalIterators(usesLocalIterators(implementingClass, conf)).setOfflineScan(isOfflineScan(implementingClass, conf));
+ return Maps.immutableEntry(tableName, queryConfig);
+ }
+ return null;
+ }
+
+ public static Map<String,Map<KeyExtent,List<Range>>> binOffline(String tableId, List<Range> ranges, Instance instance, Connector conn)
+ throws AccumuloException, TableNotFoundException {
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
+
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(instance);
+ if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
+ Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ KeyExtent lastExtent = null;
+ while (rowIter.hasNext()) {
+ Iterator<Map.Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Map.Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily().equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
+ location = entry.getValue().toString();
+ }
+
+ if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().toString().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<KeyExtent,List<Range>>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<Range>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+ return binnedRanges;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
index 727971a,0000000..13b67d5
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
@@@ -1,204 -1,0 +1,204 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
- import java.nio.charset.StandardCharsets;
-
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * @since 1.6.0
+ */
+public class OutputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link BatchWriter}.
+ *
+ * @since 1.6.0
+ */
+ public static enum WriteOpts {
+ DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ CAN_CREATE_TABLES, SIMULATION_MODE
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and
+ * underscores.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setDefaultTableName(Class<?> implementingClass, Configuration conf, String tableName) {
+ if (tableName != null)
+ conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the default table name
+ * @since 1.6.0
+ * @see #setDefaultTableName(Class, Configuration, String)
+ */
+ public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new {@link BatchWriterConfig}, with sensible built-in defaults is
+ * used. Setting the configuration multiple times overwrites any previous configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.6.0
+ */
+ public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf, BatchWriterConfig bwConfig) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String serialized;
+ try {
+ bwConfig.write(new DataOutputStream(baos));
- serialized = new String(baos.toByteArray(), StandardCharsets.UTF_8);
++ serialized = new String(baos.toByteArray(), UTF_8);
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the configuration object
+ * @since 1.6.0
+ * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
+ */
+ public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
+ String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
+ BatchWriterConfig bwConfig = new BatchWriterConfig();
+ if (serialized == null || serialized.isEmpty()) {
+ return bwConfig;
+ } else {
+ try {
- ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(StandardCharsets.UTF_8));
++ ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
+ bwConfig.readFields(new DataInputStream(bais));
+ bais.close();
+ return bwConfig;
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ }
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setCreateTables(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.6.0
+ * @see #setCreateTables(Class, Configuration, boolean)
+ */
+ public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setSimulationMode(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setSimulationMode(Class, Configuration, boolean)
+ */
+ public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 2950091,0000000..c0c0097
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@@ -1,135 -1,0 +1,136 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.partition;
+
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
- import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Scanner;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.mapreduce.lib.impl.DistributedCacheHelper;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Partitioner;
+
+/**
+ * Hadoop partitioner that uses ranges, and optionally sub-bins based on hashing.
+ */
+public class RangePartitioner extends Partitioner<Text,Writable> implements Configurable {
+ private static final String PREFIX = RangePartitioner.class.getName();
+ private static final String CUTFILE_KEY = PREFIX + ".cutFile";
+ private static final String NUM_SUBBINS = PREFIX + ".subBins";
+
+ private Configuration conf;
+
+ @Override
+ public int getPartition(Text key, Writable value, int numPartitions) {
+ try {
+ return findPartition(key, getCutPoints(), getNumSubBins());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ int findPartition(Text key, Text[] array, int numSubBins) {
+ // find the bin for the range, and guarantee it is positive
+ int index = Arrays.binarySearch(array, key);
+ index = index < 0 ? (index + 1) * -1 : index;
+
+ // both conditions work with numSubBins == 1, but this check is to avoid
+ // hashing, when we don't need to, for speed
+ if (numSubBins < 2)
+ return index;
+ return (key.toString().hashCode() & Integer.MAX_VALUE) % numSubBins + index * numSubBins;
+ }
+
+ private int _numSubBins = 0;
+
+ private synchronized int getNumSubBins() {
+ if (_numSubBins < 1) {
+ // get number of sub-bins and guarantee it is positive
+ _numSubBins = Math.max(1, getConf().getInt(NUM_SUBBINS, 1));
+ }
+ return _numSubBins;
+ }
+
+ private Text cutPointArray[] = null;
+
+ private synchronized Text[] getCutPoints() throws IOException {
+ if (cutPointArray == null) {
+ String cutFileName = conf.get(CUTFILE_KEY);
+ Path[] cf = DistributedCacheHelper.getLocalCacheFiles(conf);
+
+ if (cf != null) {
+ for (Path path : cf) {
+ if (path.toUri().getPath().endsWith(cutFileName.substring(cutFileName.lastIndexOf('/')))) {
+ TreeSet<Text> cutPoints = new TreeSet<Text>();
- Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()), StandardCharsets.UTF_8)));
++ Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()), UTF_8)));
+ try {
+ while (in.hasNextLine())
- cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(StandardCharsets.UTF_8))));
++ cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(UTF_8))));
+ } finally {
+ in.close();
+ }
+ cutPointArray = cutPoints.toArray(new Text[cutPoints.size()]);
+ break;
+ }
+ }
+ }
+ if (cutPointArray == null)
+ throw new FileNotFoundException(cutFileName + " not found in distributed cache");
+ }
+ return cutPointArray;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Sets the hdfs file name to use, containing a newline separated list of Base64 encoded split points that represent ranges for partitioning
+ */
+ public static void setSplitFile(Job job, String file) {
+ URI uri = new Path(file).toUri();
+ DistributedCacheHelper.addCacheFile(uri, job.getConfiguration());
+ job.getConfiguration().set(CUTFILE_KEY, uri.getPath());
+ }
+
+ /**
+ * Sets the number of random sub-bins per range
+ */
+ public static void setNumSubBins(Job job, int num) {
+ job.getConfiguration().setInt(NUM_SUBBINS, num);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index 0febb58,8216441..aa10276
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.minicluster.impl;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@@@ -105,8 -108,6 +107,7 @@@ import org.apache.zookeeper.server.ZooK
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
index a778add,30da0cba..bd0782d
--- a/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/ProxyServer.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.proxy;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
index 5f84c2f,ec1c9d2..4850c19
--- a/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.proxy;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/proxy/src/main/java/org/apache/accumulo/proxy/Util.java
----------------------------------------------------------------------
diff --cc proxy/src/main/java/org/apache/accumulo/proxy/Util.java
index 94b0709,7333539..bc0db72
--- a/proxy/src/main/java/org/apache/accumulo/proxy/Util.java
+++ b/proxy/src/main/java/org/apache/accumulo/proxy/Util.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.proxy;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.math.BigInteger;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.Random;
import org.apache.accumulo.proxy.thrift.IteratorSetting;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index e4d6264,6c40094..5c93a53
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 7169215,05f3c6c..efbd882
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -16,9 -16,10 +16,10 @@@
*/
package org.apache.accumulo.server.client;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
index a154fdc,18feb4f..2ad1c3d
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooCachePropertyAccessor.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.conf;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 0e9ee95,6d25f1d..c472581
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.conf;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 9d56265,587e38a..e4f61f9
--- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.constraints;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 1fd9aa3,7c4b8d3..45e883d
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@@ -16,15 -16,14 +16,15 @@@
*/
package org.apache.accumulo.server.init;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
import java.io.FileNotFoundException;
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
@@@ -412,8 -404,7 +412,8 @@@ public class Initialize
zoo.putPersistentData(zkInstanceRoot + Constants.ZTRACERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTERS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(StandardCharsets.UTF_8),
- zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8), NodeExistsPolicy.FAIL);
++ zoo.putPersistentData(zkInstanceRoot + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(UTF_8),
+ NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZGC, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZGC_LOCK, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZCONFIG, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index 4a66db7,002389a..e577153
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@@ -16,6 -16,7 +16,7 @@@
*/
package org.apache.accumulo.server.master;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
index 0e1f5ee,568e6f6..3f7f167
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/DeadServerList.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.master.state;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
index 8a09e75,b4ef519..f473ba3
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TServerInstance.java
@@@ -16,12 -16,13 +16,12 @@@
*/
package org.apache.accumulo.server.master.state;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
- import java.nio.charset.StandardCharsets;
--
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
@@@ -127,8 -120,8 +127,8 @@@ public class TServerInstance implement
return new Text(this.getSession());
}
- public Value asMutationValue() {
+ private Value asMutationValue() {
- return new Value(getLocation().toString().getBytes(StandardCharsets.UTF_8));
+ return new Value(getLocation().toString().getBytes(UTF_8));
}
public HostAndPort getLocation() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
index e06b58e,91be9c3..2a84e70
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.master.state;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
index 6ab4683,8f0c9c7..1bcf482
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooStore.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.master.state;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.accumulo.core.zookeeper.ZooUtil;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
index e7f06dc,15ade65..5481531
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/ZooTabletStateStore.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.master.state;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
- import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
index 05ede64,409fa62..54ca8de
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/AbstractMetricsImpl.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.metrics;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
index 902c063,4e909c0..930b634
--- a/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/monitor/LogService.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.monitor;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
index 590cbe4,0000000..8e9612a
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationUtil.java
@@@ -1,274 -1,0 +1,275 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.replication;
+
- import java.nio.charset.StandardCharsets;
++import static java.nio.charset.StandardCharsets.UTF_8;
++
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.replication.ReplicationConstants;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+public class ReplicationUtil {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationUtil.class);
+
+ private static final String REPLICATION_TARGET_PREFIX = Property.TABLE_REPLICATION_TARGET.getKey();
+
+ private ZooCache zooCache;
+
+ public ReplicationUtil() {
+ this(new ZooCache());
+ }
+
+ public ReplicationUtil(ZooCache cache) {
+ this.zooCache = cache;
+ }
+
+ public int getMaxReplicationThreads(Map<String,String> systemProperties, MasterMonitorInfo mmi) {
+ int activeTservers = mmi.getTServerInfoSize();
+
+ // The number of threads each tserver will use at most to replicate data
+ int replicationThreadsPerServer = Integer.parseInt(systemProperties.get(Property.REPLICATION_WORKER_THREADS.getKey()));
+
+ // The total number of "slots" we have to replicate data
+ return activeTservers * replicationThreadsPerServer;
+ }
+
+ public int getMaxReplicationThreads(Connector conn, MasterMonitorInfo mmi) throws AccumuloException, AccumuloSecurityException {
+ return getMaxReplicationThreads(conn.instanceOperations().getSystemConfiguration(), mmi);
+ }
+
+ /**
+ * Extract replication peers from system configuration
+ * @param systemProperties System properties, typically from Connector.instanceOperations().getSystemConfiguration()
+ * @return Configured replication peers
+ */
+ public Map<String,String> getPeers(Map<String,String> systemProperties) {
+ Map<String,String> peers = new HashMap<>();
+ String definedPeersPrefix = Property.REPLICATION_PEERS.getKey();
+
+ // Get the defined peers and what ReplicaSystem impl they're using
+ for (Entry<String,String> property : systemProperties.entrySet()) {
+ String key = property.getKey();
+ // Filter out cruft that we don't want
+ if (key.startsWith(definedPeersPrefix) && !key.startsWith(Property.REPLICATION_PEER_USER.getKey()) && !key.startsWith(Property.REPLICATION_PEER_PASSWORD.getKey())) {
+ String peerName = property.getKey().substring(definedPeersPrefix.length());
+ ReplicaSystem replica;
+ try {
+ replica = ReplicaSystemFactory.get(property.getValue());
+ } catch (Exception e) {
+ log.warn("Could not instantiate ReplicaSystem for {} with configuration {}", property.getKey(), property.getValue(), e);
+ continue;
+ }
+
+ peers.put(peerName, replica.getClass().getName());
+ }
+ }
+
+ return peers;
+ }
+
+ public Set<ReplicationTarget> getReplicationTargets(TableOperations tops) {
+ // The total set of configured targets
+ final Set<ReplicationTarget> allConfiguredTargets = new HashSet<>();
+ final Map<String,String> tableNameToId = tops.tableIdMap();
+
+ for (String table : tops.list()) {
+ if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) {
+ continue;
+ }
+ String localId = tableNameToId.get(table);
+ if (null == localId) {
+ log.trace("Could not determine ID for {}", table);
+ continue;
+ }
+
+ Iterable<Entry<String,String>> propertiesForTable;
+ try {
+ propertiesForTable = tops.getProperties(table);
+ } catch (AccumuloException e) {
+ log.debug("Could not fetch properties for " + table, e);
+ continue;
+ } catch (TableNotFoundException e) {
+ log.debug("Could not fetch properties for " + table, e);
+ continue;
+ }
+
+ for (Entry<String,String> prop : propertiesForTable) {
+ if (prop.getKey().startsWith(REPLICATION_TARGET_PREFIX)) {
+ String peerName = prop.getKey().substring(REPLICATION_TARGET_PREFIX.length());
+ String remoteIdentifier = prop.getValue();
+ ReplicationTarget target = new ReplicationTarget(peerName, remoteIdentifier, localId);
+
+ allConfiguredTargets.add(target);
+ }
+ }
+ }
+
+ return allConfiguredTargets;
+ }
+
+ public Map<ReplicationTarget,Long> getPendingReplications(Connector conn) {
+ final Map<ReplicationTarget,Long> counts = new HashMap<>();
+
+ // Read over the queued work
+ BatchScanner bs;
+ try {
+ bs = conn.createBatchScanner(ReplicationConstants.TABLE_NAME, Authorizations.EMPTY, 4);
+ } catch (TableNotFoundException e) {
+ log.debug("No replication table exists", e);
+ return counts;
+ }
+
+ bs.setRanges(Collections.singleton(new Range()));
+ WorkSection.limit(bs);
+ try {
+ Text buffer = new Text();
+ for (Entry<Key,Value> entry : bs) {
+ Key k = entry.getKey();
+ k.getColumnQualifier(buffer);
+ ReplicationTarget target = ReplicationTarget.from(buffer);
+
+ // TODO ACCUMULO-2835 once explicit lengths are tracked, we can give size-based estimates instead of just file-based
+ Long count = counts.get(target);
+ if (null == count) {
+ counts.put(target, Long.valueOf(1l));
+ } else {
+ counts.put(target, count + 1);
+ }
+ }
+ } finally {
+ bs.close();
+ }
+
+ return counts;
+ }
+
+ /**
+ * Fetches the absolute path of the file to be replicated.
+ * @param conn Accumulo Connector
+ * @param workQueuePath Root path for the Replication WorkQueue
+ * @param queueKey The Replication work queue key
+ * @return The absolute path for the file, or null if the key is no longer in ZooKeeper
+ */
+ public String getAbsolutePath(Connector conn, String workQueuePath, String queueKey) {
+ byte[] data = zooCache.get(workQueuePath + "/" + queueKey);
+ if (null != data) {
- return new String(data, StandardCharsets.UTF_8);
++ return new String(data, UTF_8);
+ }
+
+ return null;
+ }
+
+ /**
+ * Compute a progress string for the replication of the given WAL
+ * @param conn Accumulo Connector
+ * @param path Absolute path to a WAL, or null
+ * @param target ReplicationTarget the WAL is being replicated to
+ * @return A status message for a file being replicated
+ */
+ public String getProgress(Connector conn, String path, ReplicationTarget target) {
+ // We could try to grep over the table, but without knowing the full file path, we
+ // can't find the status quickly
+ String status = "Unknown";
+ if (null != path) {
+ Scanner s;
+ try {
+ s = conn.createScanner(ReplicationConstants.TABLE_NAME, Authorizations.EMPTY);
+ } catch (TableNotFoundException e) {
+ log.debug("Replication table no long exists", e);
+ return status;
+ }
+
+ s.setRange(Range.exact(path));
+ s.fetchColumn(WorkSection.NAME, target.toText());
+
+ // Fetch the work entry for this item
+ Entry<Key,Value> kv = null;
+ try {
+ kv = Iterables.getOnlyElement(s);
+ } catch (NoSuchElementException e) {
+ log.trace("Could not find status of {} replicating to {}", path, target);
+ status = "Unknown";
+ } finally {
+ s.close();
+ }
+
+ // If we found the work entry for it, try to compute some progress
+ if (null != kv) {
+ try {
+ Status stat = Status.parseFrom(kv.getValue().get());
+ if (StatusUtil.isFullyReplicated(stat)) {
+ status = "Finished";
+ } else {
+ if (stat.getInfiniteEnd()) {
+ status = stat.getBegin() + "/∞ records";
+ } else {
+ status = stat.getBegin() + "/" + stat.getEnd() + " records";
+ }
+ }
+ } catch (InvalidProtocolBufferException e) {
+ log.warn("Could not deserialize protobuf for {}", kv.getKey(), e);
+ status = "Unknown";
+ }
+ }
+ }
+
+ return status;
+ }
+
+ public Map<String,String> invert(Map<String,String> map) {
+ Map<String,String> newMap = Maps.newHashMapWithExpectedSize(map.size());
+ for(Entry<String,String> entry : map.entrySet()) {
+ newMap.put(entry.getValue(), entry.getKey());
+ }
+ return newMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
index 08407e4,258080c..fa90b3e
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityOperation.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 7989447,872fe11..33a5158
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@@ -16,6 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.security;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@@ -117,14 -118,14 +118,14 @@@ public final class SystemCredentials ex
}
// seed the config with the version and instance id, so at least it's not empty
- md.update(ServerConstants.WIRE_VERSION.toString().getBytes(StandardCharsets.UTF_8));
+ md.update(ServerConstants.WIRE_VERSION.toString().getBytes(UTF_8));
md.update(instanceIdBytes);
- for (Entry<String,String> entry : ServerConfiguration.getSiteConfiguration()) {
+ for (Entry<String,String> entry : SiteConfiguration.getInstance()) {
// only include instance properties
if (entry.getKey().startsWith(Property.INSTANCE_PREFIX.toString())) {
- md.update(entry.getKey().getBytes(StandardCharsets.UTF_8));
- md.update(entry.getValue().getBytes(StandardCharsets.UTF_8));
+ md.update(entry.getKey().getBytes(UTF_8));
+ md.update(entry.getValue().getBytes(UTF_8));
}
}
confChecksum = md.digest();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
index d46d623,fa1632c..afe21f2
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthenticator.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.security.handler;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.HashSet;
import java.util.Set;
import java.util.TreeSet;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
index bf22d1a,78597ad..0a1bd6c
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKAuthorizor.java
@@@ -16,8 -16,9 +16,9 @@@
*/
package org.apache.accumulo.server.security.handler;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
- import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
index 383c34e,41f880d..c03c40b
--- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.security.handler;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 7d27909,78f8c7b..886c28f
--- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.tables;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.security.SecurityPermission;
import java.util.Collections;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 38a96d4,e10c0c1..8bf4dc8
--- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.tablets;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.Random;
import org.apache.accumulo.core.Constants;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cfbdef13/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
index 7a77879,198b2d6..fb4cba0
--- a/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
@@@ -16,7 -16,8 +16,8 @@@
*/
package org.apache.accumulo.server.util;
- import java.nio.charset.StandardCharsets;
-import static com.google.common.base.Charsets.UTF_8;
++import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
[08/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
index b1bbb67,0000000..4f381ef
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java
@@@ -1,112 -1,0 +1,114 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+
+public class RenameTable extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private String tableId;
+ private String oldTableName;
+ private String newTableName;
+ private String namespaceId;
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return Utils.reserveNamespace(namespaceId, tid, false, true, TableOperation.RENAME) + Utils.reserveTable(tableId, tid, true, true, TableOperation.RENAME);
+ }
+
+ public RenameTable(String tableId, String oldTableName, String newTableName) throws NamespaceNotFoundException {
+ this.tableId = tableId;
+ this.oldTableName = oldTableName;
+ this.newTableName = newTableName;
+ Instance inst = HdfsZooInstance.getInstance();
+ this.namespaceId = Tables.getNamespaceId(inst, tableId);
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+
+ Instance instance = master.getInstance();
+ Pair<String,String> qualifiedOldTableName = Tables.qualify(oldTableName);
+ Pair<String,String> qualifiedNewTableName = Tables.qualify(newTableName);
+
+ // ensure no attempt is made to rename across namespaces
+ if (newTableName.contains(".") && !namespaceId.equals(Namespaces.getNamespaceId(instance, qualifiedNewTableName.getFirst())))
+ throw new ThriftTableOperationException(tableId, oldTableName, TableOperation.RENAME, TableOperationExceptionType.INVALID_NAME,
+ "Namespace in new table name does not match the old table name");
+
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+
+ Utils.tableNameLock.lock();
+ try {
+ Utils.checkTableDoesNotExist(instance, newTableName, tableId, TableOperation.RENAME);
+
+ final String newName = qualifiedNewTableName.getSecond();
+ final String oldName = qualifiedOldTableName.getSecond();
+
+ final String tap = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME;
+
+ zoo.mutate(tap, null, null, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] current) throws Exception {
- final String currentName = new String(current, Constants.UTF8);
++ final String currentName = new String(current, UTF_8);
+ if (currentName.equals(newName))
+ return null; // assume in this case the operation is running again, so we are done
+ if (!currentName.equals(oldName)) {
+ throw new ThriftTableOperationException(null, oldTableName, TableOperation.RENAME, TableOperationExceptionType.NOTFOUND,
+ "Name changed while processing");
+ }
- return newName.getBytes(Constants.UTF8);
++ return newName.getBytes(UTF_8);
+ }
+ });
+ Tables.clearCache(instance);
+ } finally {
+ Utils.tableNameLock.unlock();
+ Utils.unreserveTable(tableId, tid, true);
+ Utils.unreserveNamespace(this.namespaceId, tid, false);
+ }
+
+ Logger.getLogger(RenameTable.class).debug("Renamed table " + tableId + " " + oldTableName + " " + newTableName);
+
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master env) throws Exception {
+ Utils.unreserveTable(tableId, tid, true);
+ Utils.unreserveNamespace(namespaceId, tid, false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 7ede305,0000000..3e8704e
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -1,163 -1,0 +1,165 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tableOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.math.BigInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.thrift.TableOperation;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
+import org.apache.accumulo.fate.zookeeper.ZooReservation;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.zookeeper.ZooQueueLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class Utils {
+ private static final byte[] ZERO_BYTE = new byte[] {'0'};
+
+ static void checkTableDoesNotExist(Instance instance, String tableName, String tableId, TableOperation operation) throws ThriftTableOperationException {
+
+ String id = Tables.getNameToIdMap(instance).get(tableName);
+
+ if (id != null && !id.equals(tableId))
+ throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.EXISTS, null);
+ }
+
+ static String getNextTableId(String tableName, Instance instance) throws ThriftTableOperationException {
+
+ String tableId = null;
+ try {
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ final String ntp = ZooUtil.getRoot(instance) + Constants.ZTABLES;
+ byte[] nid = zoo.mutate(ntp, ZERO_BYTE, ZooUtil.PUBLIC, new Mutator() {
+ @Override
+ public byte[] mutate(byte[] currentValue) throws Exception {
- BigInteger nextId = new BigInteger(new String(currentValue, Constants.UTF8), Character.MAX_RADIX);
++ BigInteger nextId = new BigInteger(new String(currentValue, UTF_8), Character.MAX_RADIX);
+ nextId = nextId.add(BigInteger.ONE);
- return nextId.toString(Character.MAX_RADIX).getBytes(Constants.UTF8);
++ return nextId.toString(Character.MAX_RADIX).getBytes(UTF_8);
+ }
+ });
- return new String(nid, Constants.UTF8);
++ return new String(nid, UTF_8);
+ } catch (Exception e1) {
+ Logger.getLogger(CreateTable.class).error("Failed to assign tableId to " + tableName, e1);
+ throw new ThriftTableOperationException(tableId, tableName, TableOperation.CREATE, TableOperationExceptionType.OTHER, e1.getMessage());
+ }
+ }
+
+ static final Lock tableNameLock = new ReentrantLock();
+ static final Lock idLock = new ReentrantLock();
+ private static final Logger log = Logger.getLogger(Utils.class);
+
+ public static long reserveTable(String tableId, long tid, boolean writeLock, boolean tableMustExist, TableOperation op) throws Exception {
+ if (getLock(tableId, tid, writeLock).tryLock()) {
+ if (tableMustExist) {
+ Instance instance = HdfsZooInstance.getInstance();
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId))
+ throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist");
+ }
+ log.info("table " + tableId + " (" + Long.toHexString(tid) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
+ return 0;
+ } else
+ return 100;
+ }
+
+ public static void unreserveTable(String tableId, long tid, boolean writeLock) throws Exception {
+ getLock(tableId, tid, writeLock).unlock();
+ log.info("table " + tableId + " (" + Long.toHexString(tid) + ") unlocked for " + (writeLock ? "write" : "read"));
+ }
+
+ public static void unreserveNamespace(String namespaceId, long id, boolean writeLock) throws Exception {
+ getLock(namespaceId, id, writeLock).unlock();
+ log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") unlocked for " + (writeLock ? "write" : "read"));
+ }
+
+ public static long reserveNamespace(String namespaceId, long id, boolean writeLock, boolean mustExist, TableOperation op) throws Exception {
+ if (getLock(namespaceId, id, writeLock).tryLock()) {
+ if (mustExist) {
+ Instance instance = HdfsZooInstance.getInstance();
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + namespaceId))
+ throw new ThriftTableOperationException(namespaceId, "", op, TableOperationExceptionType.NAMESPACE_NOTFOUND, "Namespace does not exist");
+ }
+ log.info("namespace " + namespaceId + " (" + Long.toHexString(id) + ") locked for " + (writeLock ? "write" : "read") + " operation: " + op);
+ return 0;
+ } else
+ return 100;
+ }
+
+ public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
+ Instance instance = HdfsZooInstance.getInstance();
+
+ String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
- + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++ + Base64.encodeBase64String(directory.getBytes(UTF_8));
+
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+
+ if (ZooReservation.attempt(zk, resvPath, String.format("%016x", tid), "")) {
+ return 0;
+ } else
+ return 50;
+ }
+
+ public static void unreserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
+ Instance instance = HdfsZooInstance.getInstance();
+ String resvPath = ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS + "/"
- + Base64.encodeBase64String(directory.getBytes(Constants.UTF8));
++ + Base64.encodeBase64String(directory.getBytes(UTF_8));
+ ZooReservation.release(ZooReaderWriter.getInstance(), resvPath, String.format("%016x", tid));
+ }
+
+ private static Lock getLock(String tableId, long tid, boolean writeLock) throws Exception {
- byte[] lockData = String.format("%016x", tid).getBytes(Constants.UTF8);
++ byte[] lockData = String.format("%016x", tid).getBytes(UTF_8);
+ ZooQueueLock qlock = new ZooQueueLock(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLE_LOCKS + "/" + tableId, false);
+ Lock lock = DistributedReadWriteLock.recoverLock(qlock, lockData);
+ if (lock == null) {
+ DistributedReadWriteLock locker = new DistributedReadWriteLock(qlock, lockData);
+ if (writeLock)
+ lock = locker.writeLock();
+ else
+ lock = locker.readLock();
+ }
+ return lock;
+ }
+
+ public static Lock getReadLock(String tableId, long tid) throws Exception {
+ return Utils.getLock(tableId, tid, false);
+ }
+
+ static void checkNamespaceDoesNotExist(Instance instance, String namespace, String namespaceId, TableOperation operation)
+ throws ThriftTableOperationException {
+
+ String n = Namespaces.getNameToIdMap(instance).get(namespace);
+
+ if (n != null && !n.equals(namespaceId))
+ throw new ThriftTableOperationException(null, namespace, operation, TableOperationExceptionType.NAMESPACE_EXISTS, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
index 20b7328,0000000..a9058b7
mode 100644,000000..100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tserverOps/ShutdownTServer.java
@@@ -1,92 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.tserverOps;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.Repo;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.master.EventCoordinator.Listener;
+import org.apache.accumulo.master.tableOps.MasterRepo;
+import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransportException;
+
+public class ShutdownTServer extends MasterRepo {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger log = Logger.getLogger(ShutdownTServer.class);
+ private TServerInstance server;
+ private boolean force;
+
+ public ShutdownTServer(TServerInstance server, boolean force) {
+ this.server = server;
+ this.force = force;
+ }
+
+ @Override
+ public long isReady(long tid, Master environment) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public Repo<Master> call(long tid, Master master) throws Exception {
+ // suppress assignment of tablets to the server
+ if (force) {
+ String path = ZooUtil.getRoot(master.getInstance()) + Constants.ZTSERVERS + "/" + server.getLocation();
+ ZooLock.deleteLock(path);
+ path = ZooUtil.getRoot(master.getInstance()) + Constants.ZDEADTSERVERS + "/" + server.getLocation();
+ IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- zoo.putPersistentData(path, "forced down".getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE);
++ zoo.putPersistentData(path, "forced down".getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ return null;
+ }
+
+ // TODO move this to isReady() and drop while loop? - ACCUMULO-1259
+ Listener listener = master.getEventCoordinator().getListener();
+ master.shutdownTServer(server);
+ while (master.onlineTabletServers().contains(server)) {
+ TServerConnection connection = master.getConnection(server);
+ if (connection != null) {
+ try {
+ TabletServerStatus status = connection.getTableMap(false);
+ if (status.tableMap != null && status.tableMap.isEmpty()) {
+ log.info("tablet server hosts no tablets " + server);
+ connection.halt(master.getMasterLock());
+ log.info("tablet server asked to halt " + server);
+ break;
+ }
+ } catch (TTransportException ex) {
+ // expected
+ } catch (Exception ex) {
+ log.error("Error talking to tablet server " + server + ": " + ex);
+ }
+ }
+ listener.waitForEvents(1000);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void undo(long tid, Master m) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index c4ab6e6,0000000..75d1e9e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -1,817 -1,0 +1,819 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService;
+import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.monitor.servlets.DefaultServlet;
+import org.apache.accumulo.monitor.servlets.GcStatusServlet;
+import org.apache.accumulo.monitor.servlets.JSONServlet;
+import org.apache.accumulo.monitor.servlets.LogServlet;
+import org.apache.accumulo.monitor.servlets.MasterServlet;
+import org.apache.accumulo.monitor.servlets.OperationServlet;
+import org.apache.accumulo.monitor.servlets.ProblemServlet;
+import org.apache.accumulo.monitor.servlets.ScanServlet;
+import org.apache.accumulo.monitor.servlets.ShellServlet;
+import org.apache.accumulo.monitor.servlets.TServersServlet;
+import org.apache.accumulo.monitor.servlets.TablesServlet;
+import org.apache.accumulo.monitor.servlets.VisServlet;
+import org.apache.accumulo.monitor.servlets.XMLServlet;
+import org.apache.accumulo.monitor.servlets.trace.ListType;
+import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
+import org.apache.accumulo.monitor.servlets.trace.Summary;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.monitor.LogService;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.TableInfoUtil;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * Serve master statistics with an embedded web server.
+ */
+public class Monitor {
+ private static final Logger log = Logger.getLogger(Monitor.class);
+
+ private static final int REFRESH_TIME = 5;
+ private static long lastRecalc = 0L;
+ private static double totalIngestRate = 0.0;
+ private static double totalIngestByteRate = 0.0;
+ private static double totalQueryRate = 0.0;
+ private static double totalScanRate = 0.0;
+ private static double totalQueryByteRate = 0.0;
+ private static long totalEntries = 0L;
+ private static int totalTabletCount = 0;
+ private static int onlineTabletCount = 0;
+ private static long totalHoldTime = 0;
+ private static long totalLookups = 0;
+ private static int totalTables = 0;
+
+ private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
+ private static final long serialVersionUID = 1L;
+
+ private long maxDelta;
+
+ public MaxList(long maxDelta) {
+ this.maxDelta = maxDelta;
+ }
+
+ @Override
+ public boolean add(Pair<Long,T> obj) {
+ boolean result = super.add(obj);
+
+ if (obj.getFirst() - get(0).getFirst() > maxDelta)
+ remove(0);
+
+ return result;
+ }
+
+ }
+
+ private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
+ private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static EventCounter lookupRateTracker = new EventCounter();
+ private static EventCounter indexCacheHitTracker = new EventCounter();
+ private static EventCounter indexCacheRequestTracker = new EventCounter();
+ private static EventCounter dataCacheHitTracker = new EventCounter();
+ private static EventCounter dataCacheRequestTracker = new EventCounter();
+
+ private static volatile boolean fetching = false;
+ private static MasterMonitorInfo mmi;
+ private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
+ private static Exception problemException;
+ private static GCStatus gcStatus;
+
+ private static Instance instance;
+
+ private static ServerConfiguration config;
+
+ private static EmbeddedWebServer server;
+
+ private ZooLock monitorLock;
+
+ private static class EventCounter {
+
+ Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
+ Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
+ Set<String> serversUpdated = new HashSet<String>();
+
+ void startingUpdates() {
+ serversUpdated.clear();
+ }
+
+ void updateTabletServer(String name, long sampleTime, long numEvents) {
+ Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
+ Pair<Long,Long> lastSample = samples.get(name);
+
+ if (lastSample == null || !lastSample.equals(newSample)) {
+ samples.put(name, newSample);
+ if (lastSample != null) {
+ prevSamples.put(name, lastSample);
+ }
+ }
+ serversUpdated.add(name);
+ }
+
+ void finishedUpdating() {
+ // remove any tablet servers not updated
+ samples.keySet().retainAll(serversUpdated);
+ prevSamples.keySet().retainAll(serversUpdated);
+ }
+
+ double calculateRate() {
+ double totalRate = 0;
+
+ for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+ Pair<Long,Long> prevSample = entry.getValue();
+ Pair<Long,Long> sample = samples.get(entry.getKey());
+
+ totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
+ }
+
+ return totalRate;
+ }
+
+ long calculateCount() {
+ long count = 0;
+
+ for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+ Pair<Long,Long> prevSample = entry.getValue();
+ Pair<Long,Long> sample = samples.get(entry.getKey());
+
+ count += sample.getSecond() - prevSample.getSecond();
+ }
+
+ return count;
+ }
+ }
+
+ public static void fetchData() {
+ double totalIngestRate = 0.;
+ double totalIngestByteRate = 0.;
+ double totalQueryRate = 0.;
+ double totalQueryByteRate = 0.;
+ double totalScanRate = 0.;
+ long totalEntries = 0;
+ int totalTabletCount = 0;
+ int onlineTabletCount = 0;
+ long totalHoldTime = 0;
+ long totalLookups = 0;
+ boolean retry = true;
+
+ // only recalc every so often
+ long currentTime = System.currentTimeMillis();
+ if (currentTime - lastRecalc < REFRESH_TIME * 1000)
+ return;
+
+ synchronized (Monitor.class) {
+ if (fetching)
+ return;
+ fetching = true;
+ }
+
+ try {
+ while (retry) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnection(HdfsZooInstance.getInstance());
+ if (client != null) {
+ mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
+ retry = false;
+ } else {
+ mmi = null;
+ }
+ Monitor.gcStatus = fetchGcStatus();
+ } catch (Exception e) {
+ mmi = null;
+ log.info("Error fetching stats: " + e);
+ } finally {
+ if (client != null) {
+ MasterClient.close(client);
+ }
+ }
+ if (mmi == null)
+ UtilWaitThread.sleep(1000);
+ }
+ if (mmi != null) {
+ int majorCompactions = 0;
+ int minorCompactions = 0;
+
+ lookupRateTracker.startingUpdates();
+ indexCacheHitTracker.startingUpdates();
+ indexCacheRequestTracker.startingUpdates();
+ dataCacheHitTracker.startingUpdates();
+ dataCacheRequestTracker.startingUpdates();
+
+ for (TabletServerStatus server : mmi.tServerInfo) {
+ TableInfo summary = TableInfoUtil.summarizeTableStats(server);
+ totalIngestRate += summary.ingestRate;
+ totalIngestByteRate += summary.ingestByteRate;
+ totalQueryRate += summary.queryRate;
+ totalScanRate += summary.scanRate;
+ totalQueryByteRate += summary.queryByteRate;
+ totalEntries += summary.recs;
+ totalHoldTime += server.holdTime;
+ totalLookups += server.lookups;
+ majorCompactions += summary.majors.running;
+ minorCompactions += summary.minors.running;
+ lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
+ indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
+ indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
+ dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
+ dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
+ }
+
+ lookupRateTracker.finishedUpdating();
+ indexCacheHitTracker.finishedUpdating();
+ indexCacheRequestTracker.finishedUpdating();
+ dataCacheHitTracker.finishedUpdating();
+ dataCacheRequestTracker.finishedUpdating();
+
+ int totalTables = 0;
+ for (TableInfo tInfo : mmi.tableMap.values()) {
+ totalTabletCount += tInfo.tablets;
+ onlineTabletCount += tInfo.onlineTablets;
+ totalTables++;
+ }
+ Monitor.totalIngestRate = totalIngestRate;
+ Monitor.totalTables = totalTables;
+ totalIngestByteRate = totalIngestByteRate / 1000000.0;
+ Monitor.totalIngestByteRate = totalIngestByteRate;
+ Monitor.totalQueryRate = totalQueryRate;
+ Monitor.totalScanRate = totalScanRate;
+ totalQueryByteRate = totalQueryByteRate / 1000000.0;
+ Monitor.totalQueryByteRate = totalQueryByteRate;
+ Monitor.totalEntries = totalEntries;
+ Monitor.totalTabletCount = totalTabletCount;
+ Monitor.onlineTabletCount = onlineTabletCount;
+ Monitor.totalHoldTime = totalHoldTime;
+ Monitor.totalLookups = totalLookups;
+
+ ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
+ ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
+
+ double totalLoad = 0.;
+ for (TabletServerStatus status : mmi.tServerInfo) {
+ if (status != null)
+ totalLoad += status.osLoad;
+ }
+ loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
+
+ minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
+ majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
+
+ lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
+
+ queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
+ queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
+
+ scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
+
+ calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
+ calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
+ }
+ try {
+ Monitor.problemSummary = ProblemReports.getInstance().summarize();
+ Monitor.problemException = null;
+ } catch (Exception e) {
+ log.info("Failed to obtain problem reports ", e);
+ Monitor.problemSummary = Collections.emptyMap();
+ Monitor.problemException = e;
+ }
+
+ } finally {
+ synchronized (Monitor.class) {
+ fetching = false;
+ lastRecalc = currentTime;
+ }
+ }
+ }
+
+ private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
+ long req = cacheReq.calculateCount();
+ if (req > 0)
+ hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
+ else
+ hitRate.add(new Pair<Long,Double>(currentTime, null));
+ }
+
+ private static GCStatus fetchGcStatus() {
+ GCStatus result = null;
+ HostAndPort address = null;
+ try {
+ // Read the gc location from its lock
+ ZooReaderWriter zk = ZooReaderWriter.getInstance();
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ List<String> locks = zk.getChildren(path, null);
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
- address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), Constants.UTF8)).getAddress(Service.GC_CLIENT);
++ address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), UTF_8)).getAddress(Service.GC_CLIENT);
+ GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
+ try {
+ result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+ } catch (Exception ex) {
+ log.warn("Unable to contact the garbage collector at " + address, ex);
+ }
+ return result;
+ }
+
+ public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+
+ ServerOpts opts = new ServerOpts();
+ final String app = "monitor";
+ opts.parseArgs(app, args);
+ String hostname = opts.getAddress();
+
+ Accumulo.setupLogging(app);
+ VolumeManager fs = VolumeManagerImpl.get();
+ instance = HdfsZooInstance.getInstance();
+ config = new ServerConfiguration(instance);
+ Accumulo.init(fs, config, app);
+ Monitor monitor = new Monitor();
+ Accumulo.enableTracing(hostname, app);
+ monitor.run(hostname);
+ }
+
+ private static long START_TIME;
+
+ public void run(String hostname) {
+ try {
+ getMonitorLock();
+ } catch (Exception e) {
+ log.error("Failed to get Monitor ZooKeeper lock");
+ throw new RuntimeException(e);
+ }
+
+ Monitor.START_TIME = System.currentTimeMillis();
+ int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
+ try {
+ log.debug("Creating monitor on port " + port);
+ server = new EmbeddedWebServer(hostname, port);
+ } catch (Throwable ex) {
+ log.error("Unable to start embedded web server", ex);
+ throw new RuntimeException(ex);
+ }
+
+ server.addServlet(DefaultServlet.class, "/");
+ server.addServlet(OperationServlet.class, "/op");
+ server.addServlet(MasterServlet.class, "/master");
+ server.addServlet(TablesServlet.class, "/tables");
+ server.addServlet(TServersServlet.class, "/tservers");
+ server.addServlet(ProblemServlet.class, "/problems");
+ server.addServlet(GcStatusServlet.class, "/gc");
+ server.addServlet(LogServlet.class, "/log");
+ server.addServlet(XMLServlet.class, "/xml");
+ server.addServlet(JSONServlet.class, "/json");
+ server.addServlet(VisServlet.class, "/vis");
+ server.addServlet(ScanServlet.class, "/scans");
+ server.addServlet(Summary.class, "/trace/summary");
+ server.addServlet(ListType.class, "/trace/listType");
+ server.addServlet(ShowTrace.class, "/trace/show");
+ if (server.isUsingSsl())
+ server.addServlet(ShellServlet.class, "/shell");
+ server.start();
+
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+
+ log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
+
+ String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
+
- ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(Constants.UTF8),
++ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR_HTTP_ADDR, monitorAddress.getBytes(UTF_8),
+ NodeExistsPolicy.OVERWRITE);
+ log.info("Set monitor address in zookeeper to " + monitorAddress);
+ } catch (Exception ex) {
+ log.error("Unable to set monitor HTTP address in zookeeper", ex);
+ }
+
+ if (null != hostname) {
+ LogService.startLogListener(Monitor.getSystemConfiguration(), instance.getInstanceID(), hostname);
+ } else {
+ log.warn("Not starting log4j listener as we could not determine address to use");
+ }
+
+ new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
+
+ // need to regularly fetch data so plot data is updated
+ new Daemon(new LoggingRunnable(log, new Runnable() {
+
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Monitor.fetchData();
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+
+ UtilWaitThread.sleep(333);
+ }
+
+ }
+ }), "Data fetcher").start();
+
+ new Daemon(new LoggingRunnable(log, new Runnable() {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Monitor.fetchScans();
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ UtilWaitThread.sleep(5000);
+ }
+ }
+ }), "Scan scanner").start();
+ }
+
+ public static class ScanStats {
+ public final List<ActiveScan> scans;
+ public final long fetched;
+ ScanStats(List<ActiveScan> active) {
+ this.scans = active;
+ this.fetched = System.currentTimeMillis();
+ }
+ }
+ static final Map<String, ScanStats> allScans = new HashMap<String, ScanStats>();
+ public static Map<String, ScanStats> getScans() {
+ synchronized (allScans) {
+ return new TreeMap<String, ScanStats>(allScans);
+ }
+ }
+
+ protected static void fetchScans() throws Exception {
+ if (instance == null)
+ return;
+ Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ for (String server : c.instanceOperations().getTabletServers()) {
+ Client tserver = ThriftUtil.getTServerClient(server, Monitor.getSystemConfiguration());
+ try {
+ List<ActiveScan> scans = tserver.getActiveScans(null, SystemCredentials.get().toThrift(instance));
+ synchronized (allScans) {
+ allScans.put(server, new ScanStats(scans));
+ }
+ } catch (Exception ex) {
+ log.debug(ex, ex);
+ } finally {
+ ThriftUtil.returnClient(tserver);
+ }
+ }
+ // Age off old scan information
+ Iterator<Entry<String,ScanStats>> entryIter = allScans.entrySet().iterator();
+ long now = System.currentTimeMillis();
+ while (entryIter.hasNext()) {
+ Entry<String,ScanStats> entry = entryIter.next();
+ if (now - entry.getValue().fetched > 5 * 60 * 1000) {
+ entryIter.remove();
+ }
+ }
+ }
+
+ /**
+ * Get the monitor lock in ZooKeeper
+ */
+ private void getMonitorLock() throws KeeperException, InterruptedException {
+ final String zRoot = ZooUtil.getRoot(instance);
+ final String monitorPath = zRoot + Constants.ZMONITOR;
+ final String monitorLockPath = zRoot + Constants.ZMONITOR_LOCK;
+
+ // Ensure that everything is kosher with ZK as this has changed.
+ ZooReaderWriter zoo = ZooReaderWriter.getInstance();
+ if (zoo.exists(monitorPath)) {
+ byte[] data = zoo.getData(monitorPath, null);
+ // If the node isn't empty, it's from a previous install (has hostname:port for HTTP server)
+ if (0 != data.length) {
+ // Recursively delete from that parent node
+ zoo.recursiveDelete(monitorPath, NodeMissingPolicy.SKIP);
+
+ // And then make the nodes that we expect for the incoming ephemeral nodes
+ zoo.putPersistentData(monitorPath, new byte[0], NodeExistsPolicy.FAIL);
+ zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
+ } else if (!zoo.exists(monitorLockPath)) {
+ // monitor node in ZK exists and is empty as we expect
+ // but the monitor/lock node does not
+ zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
+ }
+ } else {
+ // 1.5.0 and earlier
+ zoo.putPersistentData(zRoot + Constants.ZMONITOR, new byte[0], NodeExistsPolicy.FAIL);
+ if (!zoo.exists(monitorLockPath)) {
+ // Somehow the monitor node exists but not monitor/lock
+ zoo.putPersistentData(monitorLockPath, new byte[0], NodeExistsPolicy.FAIL);
+ }
+ }
+
+ // Get a ZooLock for the monitor
+ while (true) {
+ MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher();
+ monitorLock = new ZooLock(monitorLockPath);
+ monitorLock.lockAsync(monitorLockWatcher, new byte[0]);
+
+ monitorLockWatcher.waitForChange();
+
+ if (monitorLockWatcher.acquiredLock) {
+ break;
+ }
+
+ if (!monitorLockWatcher.failedToAcquireLock) {
+ throw new IllegalStateException("monitor lock in unknown state");
+ }
+
+ monitorLock.tryToCancelAsyncLockOrUnlock();
+
+ UtilWaitThread.sleep(getSystemConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));
+ }
+
+ log.info("Got Monitor lock.");
+ }
+
+ /**
+ * Async Watcher for monitor lock
+ */
+ private static class MoniterLockWatcher implements ZooLock.AsyncLockWatcher {
+
+ boolean acquiredLock = false;
+ boolean failedToAcquireLock = false;
+
+ @Override
+ public void lostLock(LockLossReason reason) {
+ Halt.halt("Monitor lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
+ }
+
+ @Override
+ public void unableToMonitorLockNode(final Throwable e) {
+ Halt.halt(-1, new Runnable() {
+ @Override
+ public void run() {
+ log.fatal("No longer able to monitor Monitor lock node", e);
+ }
+ });
+
+ }
+
+ @Override
+ public synchronized void acquiredLock() {
+ if (acquiredLock || failedToAcquireLock) {
+ Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ acquiredLock = true;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void failedToAcquireLock(Exception e) {
+ log.warn("Failed to get monitor lock " + e);
+
+ if (acquiredLock) {
+ Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " + failedToAcquireLock, -1);
+ }
+
+ failedToAcquireLock = true;
+ notifyAll();
+ }
+
+ public synchronized void waitForChange() {
+ while (!acquiredLock && !failedToAcquireLock) {
+ try {
+ wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+
+ public static MasterMonitorInfo getMmi() {
+ return mmi;
+ }
+
+ public static int getTotalTables() {
+ return totalTables;
+ }
+
+ public static int getTotalTabletCount() {
+ return totalTabletCount;
+ }
+
+ public static int getOnlineTabletCount() {
+ return onlineTabletCount;
+ }
+
+ public static long getTotalEntries() {
+ return totalEntries;
+ }
+
+ public static double getTotalIngestRate() {
+ return totalIngestRate;
+ }
+
+ public static double getTotalIngestByteRate() {
+ return totalIngestByteRate;
+ }
+
+ public static double getTotalQueryRate() {
+ return totalQueryRate;
+ }
+
+ public static double getTotalScanRate() {
+ return totalScanRate;
+ }
+
+ public static double getTotalQueryByteRate() {
+ return totalQueryByteRate;
+ }
+
+ public static long getTotalHoldTime() {
+ return totalHoldTime;
+ }
+
+ public static Exception getProblemException() {
+ return problemException;
+ }
+
+ public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
+ return problemSummary;
+ }
+
+ public static GCStatus getGcStatus() {
+ return gcStatus;
+ }
+
+ public static long getTotalLookups() {
+ return totalLookups;
+ }
+
+ public static long getStartTime() {
+ return START_TIME;
+ }
+
+ public static List<Pair<Long,Double>> getLoadOverTime() {
+ synchronized (loadOverTime) {
+ return new ArrayList<Pair<Long,Double>>(loadOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIngestRateOverTime() {
+ synchronized (ingestRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
+ synchronized (ingestByteRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
+ synchronized (recoveriesOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
+ synchronized (minorCompactionsOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
+ synchronized (majorCompactionsOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getLookupsOverTime() {
+ synchronized (lookupsOverTime) {
+ return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
+ }
+ }
+
+ public static double getLookupRate() {
+ return lookupRateTracker.calculateRate();
+ }
+
+ public static List<Pair<Long,Integer>> getQueryRateOverTime() {
+ synchronized (queryRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getScanRateOverTime() {
+ synchronized (scanRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
+ synchronized (queryByteRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
+ synchronized (indexCacheHitRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
+ synchronized (dataCacheHitRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
+ }
+ }
+
+ public static AccumuloConfiguration getSystemConfiguration() {
+ return config.getConfiguration();
+ }
+
+ public static Instance getInstance() {
+ return instance;
+ }
+
+ public static boolean isUsingSsl() {
+ return server.isUsingSsl();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index 5e3e475,0000000..78dd95e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -1,140 -1,0 +1,141 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.util.Collection;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.TTimeoutTransport;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+import com.google.common.net.HostAndPort;
+
+public class ZooKeeperStatus implements Runnable {
+
+ private static final Logger log = Logger.getLogger(ZooKeeperStatus.class);
+
+ private volatile boolean stop = false;
+
+ public static class ZooKeeperState implements Comparable<ZooKeeperState> {
+ public final String keeper;
+ public final String mode;
+ public final int clients;
+
+ public ZooKeeperState(String keeper, String mode, int clients) {
+ this.keeper = keeper;
+ this.mode = mode;
+ this.clients = clients;
+ }
+
+ @Override
+ public int compareTo(ZooKeeperState other) {
+ if (this == other) {
+ return 0;
+ } else if (other == null) {
+ return 1;
+ } else {
+ if (this.keeper == other.keeper) {
+ return 0;
+ } else if (null == this.keeper) {
+ return -1;
+ } else if (null == other.keeper) {
+ return 1;
+ } else {
+ return this.keeper.compareTo(other.keeper);
+ }
+ }
+ }
+ }
+
+ private static SortedSet<ZooKeeperState> status = new TreeSet<ZooKeeperState>();
+
+ public static Collection<ZooKeeperState> getZooKeeperStatus() {
+ return status;
+ }
+
+ public void stop() {
+ this.stop = true;
+ }
+
+ @Override
+ public void run() {
+
+ while (!stop) {
+
+ TreeSet<ZooKeeperState> update = new TreeSet<ZooKeeperState>();
+
+ String zookeepers[] = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST).split(",");
+ for (String keeper : zookeepers) {
+ int clients = 0;
+ String mode = "unknown";
+
+ String[] parts = keeper.split(":");
+ TTransport transport = null;
+ try {
+ HostAndPort addr;
+ if (parts.length > 1)
+ addr = HostAndPort.fromParts(parts[0], Integer.parseInt(parts[1]));
+ else
+ addr = HostAndPort.fromParts(parts[0], 2181);
+
+ transport = TTimeoutTransport.create(addr, 10 * 1000l);
- transport.write("stat\n".getBytes(Constants.UTF8), 0, 5);
++ transport.write("stat\n".getBytes(UTF_8), 0, 5);
+ StringBuilder response = new StringBuilder();
+ try {
+ transport.flush();
+ byte[] buffer = new byte[1024 * 100];
+ int n = 0;
+ while ((n = transport.read(buffer, 0, buffer.length)) > 0) {
- response.append(new String(buffer, 0, n, Constants.UTF8));
++ response.append(new String(buffer, 0, n, UTF_8));
+ }
+ } catch (TTransportException ex) {
+ // happens at EOF
+ }
+ for (String line : response.toString().split("\n")) {
+ if (line.startsWith(" "))
+ clients++;
+ if (line.startsWith("Mode"))
+ mode = line.split(":")[1];
+ }
+ update.add(new ZooKeeperState(keeper, mode, clients));
+ } catch (Exception ex) {
+ log.info("Exception talking to zookeeper " + keeper, ex);
+ update.add(new ZooKeeperState(keeper, "Down", -1));
+ } finally {
+ if (transport != null) {
+ try {
+ transport.close();
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }
+ status = update;
+ UtilWaitThread.sleep(5 * 1000);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index d896040,0000000..7812603
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@@ -1,283 -1,0 +1,285 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.util.Date;
+import java.util.List;
+import java.util.TimerTask;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.monitor.DedupedLogEvent;
+import org.apache.accumulo.server.monitor.LogService;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+abstract public class BasicServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 1L;
+ protected static final Logger log = Logger.getLogger(BasicServlet.class);
+ static String cachedInstanceName = null;
+ private static String bannerText;
+ private static String bannerColor;
+ private static String bannerBackground;
+
+ abstract protected String getTitle(HttpServletRequest req);
+
+ @Override
+ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ StringBuilder sb = new StringBuilder();
+ try {
+ Monitor.fetchData();
+ bannerText = sanitize(Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_TEXT));
+ bannerColor = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_COLOR).replace("'", "'");
+ bannerBackground = Monitor.getSystemConfiguration().get(Property.MONITOR_BANNER_BACKGROUND).replace("'", "'");
+ pageStart(req, resp, sb);
+ pageBody(req, resp, sb);
+ pageEnd(req, resp, sb);
+ } catch (Throwable t) {
+ log.error("Error building page " + req.getRequestURI(), t);
+ sb.append("\n<pre>\n");
+ StringWriter sw = new StringWriter();
+ t.printStackTrace(new PrintWriter(sw));
+ sb.append(sanitize(sw.getBuffer().toString()));
+ sb.append("</pre>\n");
+ } finally {
+ resp.getWriter().print(sb);
+ resp.getWriter().flush();
+ }
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ doGet(req, resp);
+ }
+
+ private static final String DEFAULT_CONTENT_TYPE = "text/html";
+
+ public static final void setCookie(HttpServletResponse resp, String name, String value) {
+ resp.addCookie(new Cookie(name, value));
+ }
+
+ public static final String getCookieValue(HttpServletRequest req, String name) {
+ if (req.getCookies() != null)
+ for (Cookie c : req.getCookies())
+ if (c.getName().equals(name))
+ return c.getValue();
+ return null;
+ }
+
+ protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
+ resp.setContentType(DEFAULT_CONTENT_TYPE);
+ int refresh = -1;
+ String refreshStr = getCookieValue(req, "page.refresh.rate");
+ if (refreshStr != null) {
+ try {
+ refresh = Integer.parseInt(BasicServlet.decode(refreshStr));
+ } catch (NumberFormatException e) {
+ // ignore improperly formatted user cookie
+ }
+ }
+ synchronized (BasicServlet.class) {
+ // Learn our instance name asynchronously so we don't hang up if zookeeper is down
+ if (cachedInstanceName == null) {
+ SimpleTimer.getInstance().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ synchronized (BasicServlet.class) {
+ if (cachedInstanceName == null) {
+ cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName();
+ }
+ }
+ }
+ }, 1000);
+ }
+ }
+
+ // BEGIN PAGE
+ sb.append("<!--\n");
+ sb.append(" Licensed to the Apache Software Foundation (ASF) under one or more\n");
+ sb.append(" contributor license agreements. See the NOTICE file distributed with\n");
+ sb.append(" this work for additional information regarding copyright ownership.\n");
+ sb.append(" The ASF licenses this file to You under the Apache License, Version 2.0\n");
+ sb.append(" (the \"License\"); you may not use this file except in compliance with\n");
+ sb.append(" the License. You may obtain a copy of the License at\n");
+ sb.append("\n");
+ sb.append(" http://www.apache.org/licenses/LICENSE-2.0\n");
+ sb.append("\n");
+ sb.append(" Unless required by applicable law or agreed to in writing, software\n");
+ sb.append(" distributed under the License is distributed on an \"AS IS\" BASIS,\n");
+ sb.append(" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n");
+ sb.append(" See the License for the specific language governing permissions and\n");
+ sb.append(" limitations under the License.\n");
+ sb.append("-->\n");
+ sb.append("<html>\n");
+
+ // BEGIN HEADER
+ sb.append("<head>\n");
+ sb.append("<title>").append(getTitle(req)).append(" - Accumulo ").append(Constants.VERSION).append("</title>\n");
+ if ((refresh > 0) && (req.getRequestURI().startsWith("/vis") == false) && (req.getRequestURI().startsWith("/shell") == false))
+ sb.append("<meta http-equiv='refresh' content='" + refresh + "' />\n");
+ sb.append("<meta http-equiv='Content-Type' content='").append(DEFAULT_CONTENT_TYPE).append("' />\n");
+ sb.append("<meta http-equiv='Content-Script-Type' content='text/javascript' />\n");
+ sb.append("<meta http-equiv='Content-Style-Type' content='text/css' />\n");
+ sb.append("<link rel='shortcut icon' type='image/jpg' href='/web/favicon.png' />\n");
+ sb.append("<link rel='stylesheet' type='text/css' href='/web/screen.css' media='screen' />\n");
+ sb.append("<script src='/web/functions.js' type='text/javascript'></script>\n");
+
+ sb.append("<!--[if lte IE 8]><script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/excanvas.min.js\"></script><![endif]-->\n");
+ sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.js\"></script>\n");
+ sb.append("<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.flot.js\"></script>\n");
+
+ sb.append("</head>\n");
+
+ // BEGIN BODY OPENING
+ sb.append("\n<body ").append(getBodyAttributes()).append(">\n");
+ sb.append("<div id='content-wrapper'>\n");
+ sb.append("<div id='content'>\n");
+ sb.append("<div id='header'>");
+ if (!bannerText.isEmpty()) {
+ sb.append("<div id='banner' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
+ .append("</div>\n");
+ }
+ sb.append("<div id='headertitle'>");
+ sb.append("<h1>").append(getTitle(req)).append("</h1></div>\n");
+ sb.append("<div id='subheader'>Instance Name: ").append(cachedInstanceName).append(" Version: ").append(Constants.VERSION)
+ .append("\n");
+ sb.append("<br><span class='smalltext'>Instance ID: ").append(HdfsZooInstance.getInstance().getInstanceID()).append("</span>\n");
+ sb.append("<br><span class='smalltext'>").append(new Date().toString().replace(" ", " ")).append("</span>");
+ sb.append("</div>\n"); // end <div id='subheader'>
+ sb.append("</div>\n"); // end <div id='header'>
+
+ // BEGIN LEFT SIDE
+ sb.append("<div id='nav'>\n");
+ sb.append("<span id='nav-title'><a href='/'>Overview</a></span><br />\n");
+ sb.append("<hr />\n");
+ sb.append("<a href='/master'>Master Server</a><br />\n");
+ sb.append("<a href='/tservers'>Tablet Servers</a><br />\n");
+ sb.append("<a href='/scans'>Active Scans</a><br />\n");
+ sb.append("<a href='/vis'>Server Activity</a><br />\n");
+ sb.append("<a href='/gc'>Garbage Collector</a><br />\n");
+ sb.append("<a href='/tables'>Tables</a><br />\n");
+ sb.append("<a href='/trace/summary?minutes=10'>Recent Traces</a><br />\n");
+ List<DedupedLogEvent> dedupedLogEvents = LogService.getInstance().getEvents();
+ int numLogs = dedupedLogEvents.size();
+ boolean logsHaveError = false;
+ for (DedupedLogEvent dedupedLogEvent : dedupedLogEvents)
+ if (dedupedLogEvent.getEvent().getLevel().isGreaterOrEqual(Level.ERROR)) {
+ logsHaveError = true;
+ break;
+ }
+ if (numLogs > 0)
+ sb.append("<span class='" + (logsHaveError ? "error" : "warning") + "'><a href='/log'>Recent Logs <span class='smalltext'>(" + numLogs
+ + ")</a></span></span><br />\n");
+ int numProblems = Monitor.getProblemSummary().entrySet().size();
+ if (numProblems > 0)
+ sb.append("<span class='error'><a href='/problems'>Table Problems <span class='smalltext'>(" + numProblems + ")</a></span></span><br />\n");
+ sb.append("<hr />\n");
+ sb.append("<a href='/xml'>XML</a><br />\n");
+ sb.append("<a href='/json'>JSON</a><hr />\n");
+ if (Monitor.isUsingSsl())
+ sb.append("<a href='/shell'>Shell</a><hr />\n");
+ sb.append("<div class='smalltext'>[<a href='").append("/op?action=refresh&value=").append(refresh < 1 ? "5" : "-1");
+ sb.append("&redir=").append(currentPage(req)).append("'>");
+ sb.append(refresh < 1 ? "en" : "dis").append("able auto-refresh</a>]</div>\n");
+ sb.append("</div>\n"); // end <div id='nav'>
+
+ sb.append("<div id='main'");
+ if (bannerText.isEmpty())
+ sb.append(" style='bottom:0'");
+ sb.append(">\n");
+ sb.append("<!-- BEGIN MAIN BODY CONTENT -->\n\n");
+ }
+
+ protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
+ sb.append("This page intentionally left blank.");
+ }
+
+ protected void pageEnd(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws Exception {
+ sb.append("\n<!-- END MAIN BODY CONTENT -->\n");
+ sb.append("</div>\n"); // end <div id='main'>
+
+ // BEGIN FOOTER
+ sb.append("</div>\n"); // end <div id='content'>
+ sb.append("</div>\n"); // end <div id='content-wrapper'>
+ if (!bannerText.isEmpty()) {
+ sb.append("<div id='footer' style='color:").append(bannerColor).append(";background:").append(bannerBackground).append("'>").append(bannerText)
+ .append("</div>\n");
+ }
+ sb.append("</body>\n");
+ sb.append("</html>\n");
+ }
+
+ /**
+ * Allow the concrete servlet implementation to provide attributes on the body HTML tag, such as 'onload', which can be used to call Javascript methods on
+ * page load. By default, nothing is specified.
+ */
+ protected String getBodyAttributes() {
+ return "";
+ }
+
+ public static String encode(String s) {
+ try {
- return URLEncoder.encode(s, Constants.UTF8.name());
++ return URLEncoder.encode(s, UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
- Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++ Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
+ throw new AssertionError(e); // can't happen with UTF-8
+ }
+ }
+
+ public static String decode(String s) {
+ try {
- return URLDecoder.decode(s, Constants.UTF8.name());
++ return URLDecoder.decode(s, UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
- Logger.getLogger(BasicServlet.class).fatal(Constants.UTF8.name() + " is not a recognized encoding", e);
++ Logger.getLogger(BasicServlet.class).fatal(UTF_8.name() + " is not a recognized encoding", e);
+ throw new AssertionError(e); // can't happen with UTF-8
+ }
+ }
+
+ public static String sanitize(String xml) {
+ return xml.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
+ }
+
+ public static String currentPage(HttpServletRequest req) {
+ String redir = req.getRequestURI();
+ if (req.getQueryString() != null)
+ redir += "?" + req.getQueryString();
+ return encode(redir);
+ }
+
+ protected static void banner(StringBuilder sb, String klass, String text) {
+ sb.append("<br />\n<h2 class='").append(klass).append("'>").append(text).append("</h2>\n");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index d88bd7c,0000000..bb7e690
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -1,346 -1,0 +1,347 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
++import static com.google.common.base.Charsets.UTF_8;
++
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
- import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.util.Duration;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.ZooKeeperStatus;
+import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
+import org.apache.accumulo.monitor.util.celltypes.NumberType;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class DefaultServlet extends BasicServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "Accumulo Overview";
+ }
+
+ private void getResource(HttpServletRequest req, HttpServletResponse resp) throws IOException {
+ try {
+ String path = req.getRequestURI();
+
+ if (path.endsWith(".jpg"))
+ resp.setContentType("image/jpeg");
+
+ if (path.endsWith(".html"))
+ resp.setContentType("text/html");
+
+ path = path.substring(1);
+ InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path);
+ ServletOutputStream out = resp.getOutputStream();
+ try {
+ if (data != null) {
+ byte[] buffer = new byte[1024];
+ int n;
+ while ((n = data.read(buffer)) > 0)
+ out.write(buffer, 0, n);
+ } else {
- out.write(("could not get resource " + path + "").getBytes(Constants.UTF8));
++ out.write(("could not get resource " + path + "").getBytes(UTF_8));
+ }
+ } finally {
+ if (data != null)
+ data.close();
+ }
+ } catch (Throwable t) {
+ log.error(t, t);
+ throw new IOException(t);
+ }
+ }
+
+ @Override
+ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+ if (req.getRequestURI().startsWith("/web"))
+ getResource(req, resp);
+ else if (req.getRequestURI().startsWith("/monitor"))
+ resp.sendRedirect("/master");
+ else if (req.getRequestURI().startsWith("/errors"))
+ resp.sendRedirect("/problems");
+ else
+ super.doGet(req, resp);
+ }
+
+ public static final int GRAPH_WIDTH = 450;
+ public static final int GRAPH_HEIGHT = 150;
+
+ private static void plotData(StringBuilder sb, String title, @SuppressWarnings("rawtypes") List data, boolean points) {
+ plotData(sb, title, points, new ArrayList<String>(), data);
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static void plotData(StringBuilder sb, String title, boolean points, List<String> labels, List... series) {
+ sb.append("<div class=\"plotHeading\">");
+ sb.append(title);
+ sb.append("</div>");
+ sb.append("</br>");
+ String id = "c" + title.hashCode();
+ sb.append("<div id=\"" + id + "\" style=\"width:" + GRAPH_WIDTH + "px;height:" + GRAPH_HEIGHT + "px;\"></div>\n");
+
+ sb.append("<script type=\"text/javascript\">\n");
+ sb.append("$(function () {\n");
+
+ for (int i = 0; i < series.length; i++) {
+
+ @SuppressWarnings("unchecked")
+ List<Pair<Long,? extends Number>> data = series[i];
+ sb.append(" var d" + i + " = [");
+
+ String sep = "";
+ for (Pair<Long,? extends Number> point : data) {
+ if (point.getSecond() == null)
+ continue;
+
+ String y;
+ if (point.getSecond() instanceof Double)
+ y = String.format("%1.2f", point.getSecond());
+ else
+ y = point.getSecond().toString();
+
+ sb.append(sep);
+ sep = ",";
+ sb.append("[" + utc2local(point.getFirst()) + "," + y + "]");
+ }
+ sb.append(" ];\n");
+ }
+
+ String opts = "lines: { show: true }";
+ if (points)
+ opts = "points: { show: true, radius: 1 }";
+
+ sb.append(" $.plot($(\"#" + id + "\"),");
+ String sep = "";
+
+ String colors[] = new String[] {"red", "blue", "green", "black"};
+
+ sb.append("[");
+ for (int i = 0; i < series.length; i++) {
+ sb.append(sep);
+ sep = ",";
+ sb.append("{ ");
+ if (labels.size() > 0) {
+ sb.append("label: \"" + labels.get(i) + "\", ");
+ }
+ sb.append("data: d" + i + ", " + opts + ", color:\"" + colors[i] + "\" }");
+ }
+ sb.append("], ");
+ sb.append("{yaxis:{}, xaxis:{mode:\"time\",minTickSize: [1, \"minute\"],timeformat: \"%H:%M<br />" + getShortTZName() + "\", ticks:3}});");
+ sb.append(" });\n");
+ sb.append("</script>\n");
+ }
+
+ /**
+ * Shows the current time zone (based on the current time) short name
+ */
+ private static String getShortTZName() {
+ TimeZone tz = TimeZone.getDefault();
+ return tz.getDisplayName(tz.inDaylightTime(new Date()), TimeZone.SHORT);
+ }
+
+ /**
+ * Converts a unix timestamp in UTC to one that is relative to the local timezone
+ */
+ private static Long utc2local(Long utcMillis) {
+ Calendar currentCalendar = Calendar.getInstance(); // default timezone
+ currentCalendar.setTimeInMillis(utcMillis + currentCalendar.getTimeZone().getOffset(utcMillis));
+ return currentCalendar.getTime().getTime();
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws IOException {
+
+ sb.append("<table class='noborder'>\n");
+ sb.append("<tr>\n");
+
+ sb.append("<td class='noborder'>\n");
+ doAccumuloTable(sb);
+ sb.append("</td>\n");
+
+ sb.append("<td class='noborder'>\n");
+ doZooKeeperTable(sb);
+ sb.append("</td>\n");
+
+ sb.append("</tr></table>\n");
+ sb.append("<br/>\n");
+
+ sb.append("<p/><table class=\"noborder\">\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Ingest (Entries/s)", Monitor.getIngestRateOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Scan (Entries/s)", false, Arrays.asList("Read", "Returned"), Monitor.getScanRateOverTime(), Monitor.getQueryRateOverTime());
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Ingest (MB/s)", Monitor.getIngestByteRateOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Scan (MB/s)", Monitor.getQueryByteRateOverTime(), false);
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Load Average", Monitor.getLoadOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Seeks", Monitor.getLookupsOverTime(), false);
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Minor Compactions", Monitor.getMinorCompactionsOverTime(), false);
+ sb.append("</td><td>\n");
+ plotData(sb, "Major Compactions", Monitor.getMajorCompactionsOverTime(), false);
+ sb.append("</td></tr>\n");
+
+ sb.append("<tr><td>\n");
+ plotData(sb, "Index Cache Hit Rate", Monitor.getIndexCacheHitRateOverTime(), true);
+ sb.append("</td><td>\n");
+ plotData(sb, "Data Cache Hit Rate", Monitor.getDataCacheHitRateOverTime(), true);
+ sb.append("</td></tr>\n");
+
+ sb.append("</table>\n");
+ }
+
+ private void doAccumuloTable(StringBuilder sb) throws IOException {
+ // Accumulo
+ VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration());
+ MasterMonitorInfo info = Monitor.getMmi();
+ sb.append("<table>\n");
+ sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
+ if (info == null) {
+ sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
+ } else {
+ long totalAcuBytesUsed = 0l;
+ long totalHdfsBytesUsed = 0l;
+
+ try {
+ for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
+ final Path basePath = new Path(baseDir);
+ final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
+
+ try {
+ // Calculate the amount of space used by Accumulo on the FileSystem
+ ContentSummary accumuloSummary = fs.getContentSummary(basePath);
+ long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed();
+ totalAcuBytesUsed += bytesUsedByAcuOnFs;
+
+ // Catch the overflow -- this is big data
+ if (totalAcuBytesUsed < bytesUsedByAcuOnFs) {
+ log.debug("Overflowed long in bytes used by Accumulo for " + baseDir);
+ totalAcuBytesUsed = 0l;
+ break;
+ }
+
+ // Calculate the total amount of space used on the FileSystem
+ ContentSummary volumeSummary = fs.getContentSummary(new Path("/"));
+ long bytesUsedOnVolume = volumeSummary.getSpaceConsumed();
+ totalHdfsBytesUsed += bytesUsedOnVolume;
+
+ // Catch the overflow -- this is big data
+ if (totalHdfsBytesUsed < bytesUsedOnVolume) {
+ log.debug("Overflowed long in bytes used in HDFS for " + baseDir);
+ totalHdfsBytesUsed = 0;
+ break;
+ }
+ } catch (Exception ex) {
+ log.trace("Unable to get disk usage information for " + baseDir, ex);
+ }
+ }
+
+ String diskUsed = "Unknown";
+ String consumed = null;
+ if (totalAcuBytesUsed > 0) {
+ // Convert Accumulo usage to a readable String
+ diskUsed = bytes(totalAcuBytesUsed);
+
+ if (totalHdfsBytesUsed > 0) {
+ // Compute amount of space used by Accumulo as a percentage of total space usage.
+ consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed);
+ }
+ }
+
+ boolean highlight = false;
+ tableRow(sb, (highlight = !highlight), "Disk Used", diskUsed);
+ if (null != consumed)
+ tableRow(sb, (highlight = !highlight), "% of Used DFS", consumed);
+ tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables()));
+ tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));
+ tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Dead Tablet Servers</a>", NumberType.commas(info.deadTabletServers.size(), 0, 0));
+ tableRow(sb, (highlight = !highlight), "Tablets", NumberType.commas(Monitor.getTotalTabletCount(), 1, Long.MAX_VALUE));
+ tableRow(sb, (highlight = !highlight), "Entries", NumberType.commas(Monitor.getTotalEntries()));
+ tableRow(sb, (highlight = !highlight), "Lookups", NumberType.commas(Monitor.getTotalLookups()));
+ tableRow(sb, (highlight = !highlight), "Uptime", Duration.format(System.currentTimeMillis() - Monitor.getStartTime()));
+ } catch (Exception e) {
+ log.debug(e, e);
+ }
+ }
+ sb.append("</table>\n");
+ }
+
+ private void doZooKeeperTable(StringBuilder sb) throws IOException {
+ // Zookeepers
+ sb.append("<table>\n");
+ sb.append("<tr><th colspan='3'>Zookeeper</th></tr>\n");
+ sb.append("<tr><th>Server</th><th>Mode</th><th>Clients</th></tr>\n");
+
+ boolean highlight = false;
+ for (ZooKeeperState k : ZooKeeperStatus.getZooKeeperStatus()) {
+ if (k.clients >= 0) {
+ tableRow(sb, (highlight = !highlight), k.keeper, k.mode, k.clients);
+ } else {
+ tableRow(sb, false, k.keeper, "<span class='error'>Down</span>", "");
+ }
+ }
+ sb.append("</table>\n");
+ }
+
+ private static String bytes(long big) {
+ return NumUtil.bigNumberForSize(big);
+ }
+
+ public static void tableRow(StringBuilder sb, boolean highlight, Object... cells) {
+ sb.append(highlight ? "<tr class='highlight'>" : "<tr>");
+ for (int i = 0; i < cells.length; ++i) {
+ Object cell = cells[i];
+ String cellValue = cell == null ? "" : String.valueOf(cell).trim();
+ sb.append("<td class='").append(i < cells.length - 1 ? "left" : "right").append("'>").append(cellValue.isEmpty() ? "-" : cellValue).append("</td>");
+ }
+ sb.append("</tr>\n");
+ }
+}
[05/50] [abbrv] Merge branch '1.5' into 1.6
Posted by ct...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 03fe069,0000000..94be0bb
mode 100644,000000..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -1,3928 -1,0 +1,3929 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
++import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.CompressedIterators;
+import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletType;
+import org.apache.accumulo.core.client.impl.Translator;
+import org.apache.accumulo.core.client.impl.Translator.TKeyExtentTranslator;
+import org.apache.accumulo.core.client.impl.Translator.TRangeTranslator;
+import org.apache.accumulo.core.client.impl.Translators;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.Constraint.Environment;
+import org.apache.accumulo.core.constraints.Violations;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.ConstraintViolationSummary;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.InitialMultiScan;
+import org.apache.accumulo.core.data.thrift.InitialScan;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.data.thrift.MapFileInfo;
+import org.apache.accumulo.core.data.thrift.MultiScanResult;
+import org.apache.accumulo.core.data.thrift.ScanResult;
+import org.apache.accumulo.core.data.thrift.TCMResult;
+import org.apache.accumulo.core.data.thrift.TCMStatus;
+import org.apache.accumulo.core.data.thrift.TColumn;
+import org.apache.accumulo.core.data.thrift.TCondition;
+import org.apache.accumulo.core.data.thrift.TConditionalMutation;
+import org.apache.accumulo.core.data.thrift.TConditionalSession;
+import org.apache.accumulo.core.data.thrift.TKey;
+import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.data.thrift.TKeyValue;
+import org.apache.accumulo.core.data.thrift.TMutation;
+import org.apache.accumulo.core.data.thrift.TRange;
+import org.apache.accumulo.core.data.thrift.UpdateErrors;
+import org.apache.accumulo.core.iterators.IterationInterruptedException;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.AuthorizationContainer;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.tabletserver.thrift.ScanState;
+import org.apache.accumulo.core.tabletserver.thrift.ScanType;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Processor;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.core.util.Stat;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
+import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.ClientServiceHandler;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManager.FileType;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.log.SortedLogState;
+import org.apache.accumulo.server.master.recovery.RecoveryPath;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
+import org.apache.accumulo.server.metrics.AbstractMetricsImpl;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.security.AuditedSecurityOperation;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.FileSystemMonitor;
+import org.apache.accumulo.server.util.Halt;
+import org.apache.accumulo.server.util.MasterMetadataUtil;
+import org.apache.accumulo.server.util.MetadataTableUtil;
+import org.apache.accumulo.server.util.RpcWrapper;
+import org.apache.accumulo.server.util.TServerUtils;
+import org.apache.accumulo.server.util.TServerUtils.ServerAddress;
+import org.apache.accumulo.server.util.time.RelativeTime;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.thrift.TInfo;
+import org.apache.accumulo.tserver.Compactor.CompactionInfo;
+import org.apache.accumulo.tserver.RowLocks.RowLock;
+import org.apache.accumulo.tserver.Tablet.CommitSession;
+import org.apache.accumulo.tserver.Tablet.KVEntry;
+import org.apache.accumulo.tserver.Tablet.LookupResult;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.Tablet.ScanBatch;
+import org.apache.accumulo.tserver.Tablet.Scanner;
+import org.apache.accumulo.tserver.Tablet.SplitInfo;
+import org.apache.accumulo.tserver.Tablet.TConstraintViolationException;
+import org.apache.accumulo.tserver.Tablet.TabletClosedException;
+import org.apache.accumulo.tserver.TabletServerResourceManager.TabletResourceManager;
+import org.apache.accumulo.tserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.data.ServerConditionalMutation;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.LogSorter;
+import org.apache.accumulo.tserver.log.MutationReceiver;
+import org.apache.accumulo.tserver.log.TabletServerLogger;
+import org.apache.accumulo.tserver.mastermessage.MasterMessage;
+import org.apache.accumulo.tserver.mastermessage.SplitReportMessage;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.metrics.TabletServerMBean;
+import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
+import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.server.TServer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import com.google.common.net.HostAndPort;
+
+enum ScanRunState {
+ QUEUED, RUNNING, FINISHED
+}
+
+public class TabletServer extends AbstractMetricsImpl implements org.apache.accumulo.tserver.metrics.TabletServerMBean {
+ private static final Logger log = Logger.getLogger(TabletServer.class);
+
+ private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
+ private static long lastMemorySize = 0;
+ private static long lastMemoryCheckTime = 0;
+ private static long gcTimeIncreasedCount;
+
+ private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+ private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000;
+ private static final long TIME_BETWEEN_GC_CHECKS = 5000;
+
+ private TabletServerLogger logger;
+
+ protected TabletServerMinCMetrics mincMetrics = new TabletServerMinCMetrics();
+
+ private ServerConfiguration serverConfig;
+ private LogSorter logSorter = null;
+
+ public TabletServer(ServerConfiguration conf, VolumeManager fs) {
+ super();
+ this.serverConfig = conf;
+ this.instance = conf.getInstance();
+ this.fs = fs;
+ this.logSorter = new LogSorter(instance, fs, getSystemConfiguration());
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ synchronized (onlineTablets) {
+ long now = System.currentTimeMillis();
+ for (Tablet tablet : onlineTablets.values())
+ try {
+ tablet.updateRates(now);
+ } catch (Exception ex) {
+ log.error(ex, ex);
+ }
+ }
+ }
+ }, TIME_BETWEEN_GC_CHECKS, TIME_BETWEEN_GC_CHECKS);
+ }
+
+ private synchronized static void logGCInfo(AccumuloConfiguration conf) {
+ long now = System.currentTimeMillis();
+
+ List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ Runtime rt = Runtime.getRuntime();
+
+ StringBuilder sb = new StringBuilder("gc");
+
+ boolean sawChange = false;
+
+ long maxIncreaseInCollectionTime = 0;
+
+ for (GarbageCollectorMXBean gcBean : gcmBeans) {
+ Long prevTime = prevGcTime.get(gcBean.getName());
+ long pt = 0;
+ if (prevTime != null) {
+ pt = prevTime;
+ }
+
+ long time = gcBean.getCollectionTime();
+
+ if (time - pt != 0) {
+ sawChange = true;
+ }
+
+ long increaseInCollectionTime = time - pt;
+ sb.append(String.format(" %s=%,.2f(+%,.2f) secs", gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
+ maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
+ prevGcTime.put(gcBean.getName(), time);
+ }
+
+ long mem = rt.freeMemory();
+ if (maxIncreaseInCollectionTime == 0) {
+ gcTimeIncreasedCount = 0;
+ } else {
+ gcTimeIncreasedCount++;
+ if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
+ log.warn("Running low on memory");
+ gcTimeIncreasedCount = 0;
+ }
+ }
+
+ if (mem > lastMemorySize) {
+ sawChange = true;
+ }
+
+ String sign = "+";
+ if (mem - lastMemorySize <= 0) {
+ sign = "";
+ }
+
+ sb.append(String.format(" freemem=%,d(%s%,d) totalmem=%,d", mem, sign, (mem - lastMemorySize), rt.totalMemory()));
+
+ if (sawChange) {
+ log.debug(sb.toString());
+ }
+
+ final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) {
+ long diff = now - lastMemoryCheckTime;
+ if (diff > keepAliveTimeout) {
+ log.warn(String.format("GC pause checker not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check",
+ TIME_BETWEEN_GC_CHECKS / 1000., diff / 1000.));
+ }
+ lastMemoryCheckTime = now;
+ return;
+ }
+
+ if (maxIncreaseInCollectionTime > keepAliveTimeout) {
+ Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
+ }
+
+ lastMemorySize = mem;
+ lastMemoryCheckTime = now;
+ }
+
+ private TabletStatsKeeper statsKeeper;
+
+ private static class Session {
+ long lastAccessTime;
+ long startTime;
+ String user;
+ String client = TServerUtils.clientAddress.get();
+ public boolean reserved;
+
+ public void cleanup() {}
+ }
+
+ private static class SessionManager {
+
+ SecureRandom random;
+ Map<Long,Session> sessions;
+ long maxIdle;
+
+ SessionManager(AccumuloConfiguration conf) {
+ random = new SecureRandom();
+ sessions = new HashMap<Long,Session>();
+
+ maxIdle = conf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ sweep(maxIdle);
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, 0, Math.max(maxIdle / 2, 1000));
+ }
+
+ synchronized long createSession(Session session, boolean reserve) {
+ long sid = random.nextLong();
+
+ while (sessions.containsKey(sid)) {
+ sid = random.nextLong();
+ }
+
+ sessions.put(sid, session);
+
+ session.reserved = reserve;
+
+ session.startTime = session.lastAccessTime = System.currentTimeMillis();
+
+ return sid;
+ }
+
+ long getMaxIdleTime() {
+ return maxIdle;
+ }
+
+ /**
+ * while a session is reserved, it cannot be canceled or removed
+ */
+ synchronized Session reserveSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized Session reserveSession(long sessionId, boolean wait) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ while (wait && session.reserved) {
+ try {
+ wait(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ if (session.reserved)
+ throw new IllegalStateException();
+ session.reserved = true;
+ }
+
+ return session;
+
+ }
+
+ synchronized void unreserveSession(Session session) {
+ if (!session.reserved)
+ throw new IllegalStateException();
+ notifyAll();
+ session.reserved = false;
+ session.lastAccessTime = System.currentTimeMillis();
+ }
+
+ synchronized void unreserveSession(long sessionId) {
+ Session session = getSession(sessionId);
+ if (session != null)
+ unreserveSession(session);
+ }
+
+ synchronized Session getSession(long sessionId) {
+ Session session = sessions.get(sessionId);
+ if (session != null)
+ session.lastAccessTime = System.currentTimeMillis();
+ return session;
+ }
+
+ Session removeSession(long sessionId) {
+ return removeSession(sessionId, false);
+ }
+
+ Session removeSession(long sessionId, boolean unreserve) {
+ Session session = null;
+ synchronized (this) {
+ session = sessions.remove(sessionId);
+ if (unreserve && session != null)
+ unreserveSession(session);
+ }
+
+ // do clean up out side of lock..
+ if (session != null)
+ session.cleanup();
+
+ return session;
+ }
+
+ private void sweep(long maxIdle) {
+ ArrayList<Session> sessionsToCleanup = new ArrayList<Session>();
+ synchronized (this) {
+ Iterator<Session> iter = sessions.values().iterator();
+ while (iter.hasNext()) {
+ Session session = iter.next();
+ long idleTime = System.currentTimeMillis() - session.lastAccessTime;
+ if (idleTime > maxIdle && !session.reserved) {
+ iter.remove();
+ sessionsToCleanup.add(session);
+ }
+ }
+ }
+
+ // do clean up outside of lock
+ for (Session session : sessionsToCleanup) {
+ session.cleanup();
+ }
+ }
+
+ synchronized void removeIfNotAccessed(final long sessionId, long delay) {
+ Session session = sessions.get(sessionId);
+ if (session != null) {
+ final long removeTime = session.lastAccessTime;
+ TimerTask r = new TimerTask() {
+ @Override
+ public void run() {
+ Session sessionToCleanup = null;
+ synchronized (SessionManager.this) {
+ Session session2 = sessions.get(sessionId);
+ if (session2 != null && session2.lastAccessTime == removeTime && !session2.reserved) {
+ sessions.remove(sessionId);
+ sessionToCleanup = session2;
+ }
+ }
+
+ // call clean up outside of lock
+ if (sessionToCleanup != null)
+ sessionToCleanup.cleanup();
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(r, delay);
+ }
+ }
+
+ public synchronized Map<String,MapCounter<ScanRunState>> getActiveScansPerTable() {
+ Map<String,MapCounter<ScanRunState>> counts = new HashMap<String,MapCounter<ScanRunState>>();
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+
+ Session session = entry.getValue();
+ @SuppressWarnings("rawtypes")
+ ScanTask nbt = null;
+ String tableID = null;
+
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+ nbt = ss.nextBatchTask;
+ tableID = ss.extent.getTableId().toString();
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+ nbt = mss.lookupTask;
+ tableID = mss.threadPoolExtent.getTableId().toString();
+ }
+
+ if (nbt == null)
+ continue;
+
+ ScanRunState srs = nbt.getScanRunState();
+
+ if (srs == ScanRunState.FINISHED)
+ continue;
+
+ MapCounter<ScanRunState> stateCounts = counts.get(tableID);
+ if (stateCounts == null) {
+ stateCounts = new MapCounter<ScanRunState>();
+ counts.put(tableID, stateCounts);
+ }
+
+ stateCounts.increment(srs, 1);
+ }
+
+ return counts;
+ }
+
+ public synchronized List<ActiveScan> getActiveScans() {
+
+ ArrayList<ActiveScan> activeScans = new ArrayList<ActiveScan>();
+
+ long ct = System.currentTimeMillis();
+
+ for (Entry<Long,Session> entry : sessions.entrySet()) {
+ Session session = entry.getValue();
+ if (session instanceof ScanSession) {
+ ScanSession ss = (ScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<ScanBatch> nbt = ss.nextBatchTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(ss.client, ss.user, ss.extent.getTableId().toString(), ct - ss.startTime, ct - ss.lastAccessTime, ScanType.SINGLE,
+ state, ss.extent.toThrift(), Translator.translate(ss.columnSet, Translators.CT), ss.ssiList, ss.ssio, ss.auths.getAuthorizationsBB()));
+
+ } else if (session instanceof MultiScanSession) {
+ MultiScanSession mss = (MultiScanSession) session;
+
+ ScanState state = ScanState.RUNNING;
+
+ ScanTask<MultiScanResult> nbt = mss.lookupTask;
+ if (nbt == null) {
+ state = ScanState.IDLE;
+ } else {
+ switch (nbt.getScanRunState()) {
+ case QUEUED:
+ state = ScanState.QUEUED;
+ break;
+ case FINISHED:
+ state = ScanState.IDLE;
+ break;
+ case RUNNING:
+ default:
+ /* do nothing */
+ break;
+ }
+ }
+
+ activeScans.add(new ActiveScan(mss.client, mss.user, mss.threadPoolExtent.getTableId().toString(), ct - mss.startTime, ct - mss.lastAccessTime,
+ ScanType.BATCH, state, mss.threadPoolExtent.toThrift(), Translator.translate(mss.columnSet, Translators.CT), mss.ssiList, mss.ssio, mss.auths
+ .getAuthorizationsBB()));
+ }
+ }
+
+ return activeScans;
+ }
+ }
+
+ static class TservConstraintEnv implements Environment {
+
+ private TCredentials credentials;
+ private SecurityOperation security;
+ private Authorizations auths;
+ private KeyExtent ke;
+
+ TservConstraintEnv(SecurityOperation secOp, TCredentials credentials) {
+ this.security = secOp;
+ this.credentials = credentials;
+ }
+
+ void setExtent(KeyExtent ke) {
+ this.ke = ke;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return ke;
+ }
+
+ @Override
+ public String getUser() {
+ return credentials.getPrincipal();
+ }
+
+ @Override
+ @Deprecated
+ public Authorizations getAuthorizations() {
+ if (auths == null)
+ try {
+ this.auths = security.getUserAuthorizations(credentials);
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ return auths;
+ }
+
+ @Override
+ public AuthorizationContainer getAuthorizationsContainer() {
+ return new AuthorizationContainer() {
+ @Override
+ public boolean contains(ByteSequence auth) {
+ try {
+ return security.userHasAuthorizations(credentials,
+ Collections.<ByteBuffer> singletonList(ByteBuffer.wrap(auth.getBackingArray(), auth.offset(), auth.length())));
+ } catch (ThriftSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+ }
+
+ private abstract class ScanTask<T> implements RunnableFuture<T> {
+
+ protected AtomicBoolean interruptFlag;
+ protected ArrayBlockingQueue<Object> resultQueue;
+ protected AtomicInteger state;
+ protected AtomicReference<ScanRunState> runState;
+
+ private static final int INITIAL = 1;
+ private static final int ADDED = 2;
+ private static final int CANCELED = 3;
+
+ ScanTask() {
+ interruptFlag = new AtomicBoolean(false);
+ runState = new AtomicReference<ScanRunState>(ScanRunState.QUEUED);
+ state = new AtomicInteger(INITIAL);
+ resultQueue = new ArrayBlockingQueue<Object>(1);
+ }
+
+ protected void addResult(Object o) {
+ if (state.compareAndSet(INITIAL, ADDED))
+ resultQueue.add(o);
+ else if (state.get() == ADDED)
+ throw new IllegalStateException("Tried to add more than one result");
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!mayInterruptIfRunning)
+ throw new IllegalArgumentException("Cancel will always attempt to interupt running next batch task");
+
+ if (state.get() == CANCELED)
+ return true;
+
+ if (state.compareAndSet(INITIAL, CANCELED)) {
+ interruptFlag.set(true);
+ resultQueue = null;
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+
+ ArrayBlockingQueue<Object> localRQ = resultQueue;
+
+ if (state.get() == CANCELED)
+ throw new CancellationException();
+
+ if (localRQ == null && state.get() == ADDED)
+ throw new IllegalStateException("Tried to get result twice");
+
+ Object r = localRQ.poll(timeout, unit);
+
+ // could have been canceled while waiting
+ if (state.get() == CANCELED) {
+ if (r != null)
+ throw new IllegalStateException("Nothing should have been added when in canceled state");
+
+ throw new CancellationException();
+ }
+
+ if (r == null)
+ throw new TimeoutException();
+
+ // make this method stop working now that something is being
+ // returned
+ resultQueue = null;
+
+ if (r instanceof Throwable)
+ throw new ExecutionException((Throwable) r);
+
+ return (T) r;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return state.get() == CANCELED;
+ }
+
+ @Override
+ public boolean isDone() {
+ return runState.get().equals(ScanRunState.FINISHED);
+ }
+
+ public ScanRunState getScanRunState() {
+ return runState.get();
+ }
+
+ }
+
+ private static class ConditionalSession extends Session {
+ public TCredentials credentials;
+ public Authorizations auths;
+ public String tableId;
+ public AtomicBoolean interruptFlag;
+
+ @Override
+ public void cleanup() {
+ interruptFlag.set(true);
+ }
+ }
+
+ private static class UpdateSession extends Session {
+ public Tablet currentTablet;
+ public MapCounter<Tablet> successfulCommits = new MapCounter<Tablet>();
+ Map<KeyExtent,Long> failures = new HashMap<KeyExtent,Long>();
+ HashMap<KeyExtent,SecurityErrorCode> authFailures = new HashMap<KeyExtent,SecurityErrorCode>();
+ public Violations violations;
+ public TCredentials credentials;
+ public long totalUpdates = 0;
+ public long flushTime = 0;
+ Stat prepareTimes = new Stat();
+ Stat walogTimes = new Stat();
+ Stat commitTimes = new Stat();
+ Stat authTimes = new Stat();
+ public Map<Tablet,List<Mutation>> queuedMutations = new HashMap<Tablet,List<Mutation>>();
+ public long queuedMutationSize = 0;
+ TservConstraintEnv cenv = null;
+ }
+
+ private static class ScanSession extends Session {
+ public KeyExtent extent;
+ public HashSet<Column> columnSet;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+ public long entriesReturned = 0;
+ public Stat nbTimes = new Stat();
+ public long batchCount = 0;
+ public volatile ScanTask<ScanBatch> nextBatchTask;
+ public AtomicBoolean interruptFlag;
+ public Scanner scanner;
+ public long readaheadThreshold = Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
+
+ @Override
+ public void cleanup() {
+ try {
+ if (nextBatchTask != null)
+ nextBatchTask.cancel(true);
+ } finally {
+ if (scanner != null)
+ scanner.close();
+ }
+ }
+
+ }
+
+ private static class MultiScanSession extends Session {
+ HashSet<Column> columnSet;
+ Map<KeyExtent,List<Range>> queries;
+ public List<IterInfo> ssiList;
+ public Map<String,Map<String,String>> ssio;
+ public Authorizations auths;
+
+ // stats
+ int numRanges;
+ int numTablets;
+ int numEntries;
+ long totalLookupTime;
+
+ public volatile ScanTask<MultiScanResult> lookupTask;
+ public KeyExtent threadPoolExtent;
+
+ @Override
+ public void cleanup() {
+ if (lookupTask != null)
+ lookupTask.cancel(true);
+ }
+ }
+
+ /**
+ * This little class keeps track of writes in progress and allows readers to wait for writes that started before the read. It assumes that the operation ids
+ * are monotonically increasing.
+ *
+ */
+ static class WriteTracker {
+ private static AtomicLong operationCounter = new AtomicLong(1);
+ private Map<TabletType,TreeSet<Long>> inProgressWrites = new EnumMap<TabletType,TreeSet<Long>>(TabletType.class);
+
+ WriteTracker() {
+ for (TabletType ttype : TabletType.values()) {
+ inProgressWrites.put(ttype, new TreeSet<Long>());
+ }
+ }
+
+ synchronized long startWrite(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ inProgressWrites.get(ttype).add(operationId);
+ return operationId;
+ }
+
+ synchronized void finishWrite(long operationId) {
+ if (operationId == -1)
+ return;
+
+ boolean removed = false;
+
+ for (TabletType ttype : TabletType.values()) {
+ removed = inProgressWrites.get(ttype).remove(operationId);
+ if (removed)
+ break;
+ }
+
+ if (!removed) {
+ throw new IllegalArgumentException("Attempted to finish write not in progress, operationId " + operationId);
+ }
+
+ this.notifyAll();
+ }
+
+ synchronized void waitForWrites(TabletType ttype) {
+ long operationId = operationCounter.getAndIncrement();
+ while (inProgressWrites.get(ttype).floor(operationId) != null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ log.error(e, e);
+ }
+ }
+ }
+
+ public long startWrite(Set<Tablet> keySet) {
+ if (keySet.size() == 0)
+ return -1;
+
+ ArrayList<KeyExtent> extents = new ArrayList<KeyExtent>(keySet.size());
+
+ for (Tablet tablet : keySet)
+ extents.add(tablet.getExtent());
+
+ return startWrite(TabletType.type(extents));
+ }
+ }
+
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ TransactionWatcher watcher = new TransactionWatcher();
+
+ private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface {
+
+ SessionManager sessionManager;
+
+ AccumuloConfiguration acuConf = getSystemConfiguration();
+
+ TabletServerUpdateMetrics updateMetrics = new TabletServerUpdateMetrics();
+
+ TabletServerScanMetrics scanMetrics = new TabletServerScanMetrics();
+
+ WriteTracker writeTracker = new WriteTracker();
+
+ private RowLocks rowLocks = new RowLocks();
+
+ ThriftClientHandler() {
+ super(instance, watcher, fs);
+ log.debug(ThriftClientHandler.class.getName() + " created");
+ sessionManager = new SessionManager(getSystemConfiguration());
+ // Register the metrics MBean
+ try {
+ updateMetrics.register();
+ scanMetrics.register();
+ } catch (Exception e) {
+ log.error("Exception registering MBean with MBean Server", e);
+ }
+ }
+
+ @Override
+ public List<TKeyExtent> bulkImport(TInfo tinfo, TCredentials credentials, long tid, Map<TKeyExtent,Map<String,MapFileInfo>> files, boolean setTime)
+ throws ThriftSecurityException {
+
+ if (!security.canPerformSystemActions(credentials))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ List<TKeyExtent> failures = new ArrayList<TKeyExtent>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+ Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
+ for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
+ Path path = new Path(mapping.getKey());
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ path = ns.makeQualified(path);
+ fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
+ }
+
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileRefMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files " + fileMap.keySet() + " not imported to " + new KeyExtent(tke) + ": " + ioe.getMessage());
+ failures.add(tke);
+ }
+ }
+ }
+ return failures;
+ }
+
+ private class NextBatchTask extends ScanTask<ScanBatch> {
+
+ private long scanID;
+
+ NextBatchTask(long scanID, AtomicBoolean interruptFlag) {
+ this.scanID = scanID;
+ this.interruptFlag = interruptFlag;
+
+ if (interruptFlag.get())
+ cancel(true);
+ }
+
+ @Override
+ public void run() {
+
+ final ScanSession scanSession = (ScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || scanSession == null)
+ return;
+
+ runState.set(ScanRunState.RUNNING);
+
+ Thread.currentThread().setName(
+ "User: " + scanSession.user + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
+
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+
+ if (tablet == null) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ return;
+ }
+
+ long t1 = System.currentTimeMillis();
+ ScanBatch batch = scanSession.scanner.read();
+ long t2 = System.currentTimeMillis();
+ scanSession.nbTimes.addStat(t2 - t1);
+
+ // there should only be one thing on the queue at a time, so
+ // it should be ok to call add()
+ // instead of put()... if add() fails because queue is at
+ // capacity it means there is code
+ // problem somewhere
+ addResult(batch);
+ } catch (TabletClosedException e) {
+ addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (TooManyFilesException tmfe) {
+ addResult(tmfe);
+ } catch (Throwable e) {
+ log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
+ addResult(e);
+ } finally {
+ runState.set(ScanRunState.FINISHED);
+ Thread.currentThread().setName(oldThreadName);
+ }
+
+ }
+ }
+
+ private class LookupTask extends ScanTask<MultiScanResult> {
+
+ private long scanID;
+
+ LookupTask(long scanID) {
+ this.scanID = scanID;
+ }
+
+ @Override
+ public void run() {
+ MultiScanSession session = (MultiScanSession) sessionManager.getSession(scanID);
+ String oldThreadName = Thread.currentThread().getName();
+
+ try {
+ if (isCancelled() || session == null)
+ return;
+
+ TableConfiguration acuTableConf = ServerConfiguration.getTableConfiguration(instance, session.threadPoolExtent.getTableId().toString());
+ long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM);
+
+ runState.set(ScanRunState.RUNNING);
+ Thread.currentThread().setName("Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Table: ");
+
+ long bytesAdded = 0;
+ long maxScanTime = 4000;
+
+ long startTime = System.currentTimeMillis();
+
+ ArrayList<KVEntry> results = new ArrayList<KVEntry>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ ArrayList<KeyExtent> fullScans = new ArrayList<KeyExtent>();
+ KeyExtent partScan = null;
+ Key partNextKey = null;
+ boolean partNextKeyInclusive = false;
+
+ Iterator<Entry<KeyExtent,List<Range>>> iter = session.queries.entrySet().iterator();
+
+ // check the time so that the read ahead thread is not monopolized
+ while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
+ Entry<KeyExtent,List<Range>> entry = iter.next();
+
+ iter.remove();
+
+ // check that tablet server is serving requested tablet
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null) {
+ failures.put(entry.getKey(), entry.getValue());
+ continue;
+ }
+ Thread.currentThread().setName(
+ "Client: " + session.client + " User: " + session.user + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
+
+ LookupResult lookupResult;
+ try {
+
+ // do the following check to avoid a race condition
+ // between setting false below and the task being
+ // canceled
+ if (isCancelled())
+ interruptFlag.set(true);
+
+ lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList,
+ session.ssio, interruptFlag);
+
+ // if the tablet was closed it it possible that the
+ // interrupt flag was set.... do not want it set for
+ // the next
+ // lookup
+ interruptFlag.set(false);
+
+ } catch (IOException e) {
+ log.warn("lookup failed for tablet " + entry.getKey(), e);
+ throw new RuntimeException(e);
+ }
+
+ bytesAdded += lookupResult.bytesAdded;
+
+ if (lookupResult.unfinishedRanges.size() > 0) {
+ if (lookupResult.closed) {
+ failures.put(entry.getKey(), lookupResult.unfinishedRanges);
+ } else {
+ session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
+ partScan = entry.getKey();
+ partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
+ partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
+ }
+ } else {
+ fullScans.add(entry.getKey());
+ }
+ }
+
+ long finishTime = System.currentTimeMillis();
+ session.totalLookupTime += (finishTime - startTime);
+ session.numEntries += results.size();
+
+ // convert everything to thrift before adding result
+ List<TKeyValue> retResults = new ArrayList<TKeyValue>();
+ for (KVEntry entry : results)
+ retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
+ Map<TKeyExtent,List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<Range,TRange>(Translators.RT));
+ List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
+ TKeyExtent retPartScan = null;
+ TKey retPartNextKey = null;
+ if (partScan != null) {
+ retPartScan = partScan.toThrift();
+ retPartNextKey = partNextKey.toThrift();
+ }
+ // add results to queue
+ addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
+ } catch (IterationInterruptedException iie) {
+ if (!isCancelled()) {
+ log.warn("Iteration interrupted, when scan not cancelled", iie);
+ addResult(iie);
+ }
+ } catch (Throwable e) {
+ log.warn("exception while doing multi-scan ", e);
+ addResult(e);
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ runState.set(ScanRunState.FINISHED);
+ }
+ }
+ }
+
+ @Override
+ public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated,
+ long readaheadThreshold) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
- String tableId = new String(textent.getTable(), Constants.UTF8);
++ String tableId = new String(textent.getTable(), UTF_8);
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), range, columns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ KeyExtent extent = new KeyExtent(textent);
+
+ // wait for any writes that are in flight.. this done to ensure
+ // consistency across client restarts... assume a client writes
+ // to accumulo and dies while waiting for a confirmation from
+ // accumulo... the client process restarts and tries to read
+ // data from accumulo making the assumption that it will get
+ // any writes previously made... however if the server side thread
+ // processing the write from the dead client is still in progress,
+ // the restarted client may not see the write unless we wait here.
+ // this behavior is very important when the client is reading the
+ // metadata
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(extent));
+
+ Tablet tablet = onlineTablets.get(extent);
+ if (tablet == null)
+ throw new NotServingTabletException(textent);
+
+ ScanSession scanSession = new ScanSession();
+ scanSession.user = credentials.getPrincipal();
+ scanSession.extent = new KeyExtent(extent);
+ scanSession.columnSet = new HashSet<Column>();
+ scanSession.ssiList = ssiList;
+ scanSession.ssio = ssio;
+ scanSession.auths = new Authorizations(authorizations);
+ scanSession.interruptFlag = new AtomicBoolean();
+ scanSession.readaheadThreshold = readaheadThreshold;
+
+ for (TColumn tcolumn : columns) {
+ scanSession.columnSet.add(new Column(tcolumn));
+ }
+
+ scanSession.scanner = tablet.createScanner(new Range(range), batchSize, scanSession.columnSet, scanSession.auths, ssiList, ssio, isolated,
+ scanSession.interruptFlag);
+
+ long sid = sessionManager.createSession(scanSession, true);
+
+ ScanResult scanResult;
+ try {
+ scanResult = continueScan(tinfo, sid, scanSession);
+ } catch (NoSuchScanIDException e) {
+ log.error("The impossible happened", e);
+ throw new RuntimeException();
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialScan(sid, scanResult);
+ }
+
+ @Override
+ public ScanResult continueScan(TInfo tinfo, long scanID) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+ ScanSession scanSession = (ScanSession) sessionManager.reserveSession(scanID);
+ if (scanSession == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueScan(tinfo, scanID, scanSession);
+ } finally {
+ sessionManager.unreserveSession(scanSession);
+ }
+ }
+
+ private ScanResult continueScan(TInfo tinfo, long scanID, ScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException,
+ org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException {
+
+ if (scanSession.nextBatchTask == null) {
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ ScanBatch bresult;
+ try {
+ bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ scanSession.nextBatchTask = null;
+ } catch (ExecutionException e) {
+ sessionManager.removeSession(scanID);
+ if (e.getCause() instanceof NotServingTabletException)
+ throw (NotServingTabletException) e.getCause();
+ else if (e.getCause() instanceof TooManyFilesException)
+ throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
+ else
+ throw new RuntimeException(e);
+ } catch (CancellationException ce) {
+ sessionManager.removeSession(scanID);
+ Tablet tablet = onlineTablets.get(scanSession.extent);
+ if (tablet == null || tablet.isClosed())
+ throw new NotServingTabletException(scanSession.extent.toThrift());
+ else
+ throw new NoSuchScanIDException();
+ } catch (TimeoutException e) {
+ List<TKeyValue> param = Collections.emptyList();
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ return new ScanResult(param, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get next batch", t);
+ throw new RuntimeException(t);
+ }
+
+ ScanResult scanResult = new ScanResult(Key.compress(bresult.results), bresult.more);
+
+ scanSession.entriesReturned += scanResult.results.size();
+
+ scanSession.batchCount++;
+
+ if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
+ // start reading next batch while current batch is transmitted
+ // to client
+ scanSession.nextBatchTask = new NextBatchTask(scanID, scanSession.interruptFlag);
+ resourceManager.executeReadAhead(scanSession.extent, scanSession.nextBatchTask);
+ }
+
+ if (!scanResult.more)
+ closeScan(tinfo, scanID);
+
+ return scanResult;
+ }
+
+ @Override
+ public void closeScan(TInfo tinfo, long scanID) {
+ ScanSession ss = (ScanSession) sessionManager.removeSession(scanID);
+ if (ss != null) {
+ long t2 = System.currentTimeMillis();
+
+ log.debug(String.format("ScanSess tid %s %s %,d entries in %.2f secs, nbTimes = [%s] ", TServerUtils.clientAddress.get(), ss.extent.getTableId()
+ .toString(), ss.entriesReturned, (t2 - ss.startTime) / 1000.0, ss.nbTimes.toString()));
+ if (scanMetrics.isEnabled()) {
+ scanMetrics.add(TabletServerScanMetrics.scan, t2 - ss.startTime);
+ scanMetrics.add(TabletServerScanMetrics.resultSize, ss.entriesReturned);
+ }
+ }
+ }
+
+ @Override
+ public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns,
+ List<IterInfo> ssiList, Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites) throws ThriftSecurityException {
+ // find all of the tables that need to be scanned
+ HashSet<String> tables = new HashSet<String>();
+ for (TKeyExtent keyExtent : tbatch.keySet()) {
- tables.add(new String(keyExtent.getTable(), Constants.UTF8));
++ tables.add(new String(keyExtent.getTable(), UTF_8));
+ }
+
+ if (tables.size() != 1)
+ throw new IllegalArgumentException("Cannot batch scan over multiple tables");
+
+ // check if user has permission to the tables
+ for (String tableId : tables)
+ if (!security.canScan(credentials, tableId, Tables.getNamespaceId(instance, tableId), tbatch, tcolumns, ssiList, ssio, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ try {
+ if (!security.userHasAuthorizations(credentials, authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+ } catch (ThriftSecurityException tse) {
+ log.error(tse, tse);
+ throw tse;
+ }
+ Map<KeyExtent,List<Range>> batch = Translator.translate(tbatch, new TKeyExtentTranslator(), new Translator.ListTranslator<TRange,Range>(
+ new TRangeTranslator()));
+
+ // This is used to determine which thread pool to use
+ KeyExtent threadPoolExtent = batch.keySet().iterator().next();
+
+ if (waitForWrites)
+ writeTracker.waitForWrites(TabletType.type(batch.keySet()));
+
+ MultiScanSession mss = new MultiScanSession();
+ mss.user = credentials.getPrincipal();
+ mss.queries = batch;
+ mss.columnSet = new HashSet<Column>(tcolumns.size());
+ mss.ssiList = ssiList;
+ mss.ssio = ssio;
+ mss.auths = new Authorizations(authorizations);
+
+ mss.numTablets = batch.size();
+ for (List<Range> ranges : batch.values()) {
+ mss.numRanges += ranges.size();
+ }
+
+ for (TColumn tcolumn : tcolumns)
+ mss.columnSet.add(new Column(tcolumn));
+
+ mss.threadPoolExtent = threadPoolExtent;
+
+ long sid = sessionManager.createSession(mss, true);
+
+ MultiScanResult result;
+ try {
+ result = continueMultiScan(tinfo, sid, mss);
+ } catch (NoSuchScanIDException e) {
+ log.error("the impossible happened", e);
+ throw new RuntimeException("the impossible happened", e);
+ } finally {
+ sessionManager.unreserveSession(sid);
+ }
+
+ return new InitialMultiScan(sid, result);
+ }
+
+ @Override
+ public MultiScanResult continueMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+
+ MultiScanSession session = (MultiScanSession) sessionManager.reserveSession(scanID);
+
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ try {
+ return continueMultiScan(tinfo, scanID, session);
+ } finally {
+ sessionManager.unreserveSession(session);
+ }
+ }
+
+ private MultiScanResult continueMultiScan(TInfo tinfo, long scanID, MultiScanSession session) throws NoSuchScanIDException {
+
+ if (session.lookupTask == null) {
+ session.lookupTask = new LookupTask(scanID);
+ resourceManager.executeReadAhead(session.threadPoolExtent, session.lookupTask);
+ }
+
+ try {
+ MultiScanResult scanResult = session.lookupTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
+ session.lookupTask = null;
+ return scanResult;
+ } catch (TimeoutException e1) {
+ long timeout = acuConf.getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
+ sessionManager.removeIfNotAccessed(scanID, timeout);
+ List<TKeyValue> results = Collections.emptyList();
+ Map<TKeyExtent,List<TRange>> failures = Collections.emptyMap();
+ List<TKeyExtent> fullScans = Collections.emptyList();
+ return new MultiScanResult(results, failures, fullScans, null, null, false, true);
+ } catch (Throwable t) {
+ sessionManager.removeSession(scanID);
+ log.warn("Failed to get multiscan result", t);
+ throw new RuntimeException(t);
+ }
+ }
+
+ @Override
+ public void closeMultiScan(TInfo tinfo, long scanID) throws NoSuchScanIDException {
+ MultiScanSession session = (MultiScanSession) sessionManager.removeSession(scanID);
+ if (session == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ long t2 = System.currentTimeMillis();
+ log.debug(String.format("MultiScanSess %s %,d entries in %.2f secs (lookup_time:%.2f secs tablets:%,d ranges:%,d) ", TServerUtils.clientAddress.get(),
+ session.numEntries, (t2 - session.startTime) / 1000.0, session.totalLookupTime / 1000.0, session.numTablets, session.numRanges));
+ }
+
+ @Override
+ public long startUpdate(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
+ // Make sure user is real
+
+ security.authenticateUser(credentials, credentials);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+
+ UpdateSession us = new UpdateSession();
+ us.violations = new Violations();
+ us.credentials = credentials;
+ us.cenv = new TservConstraintEnv(security, us.credentials);
+
+ long sid = sessionManager.createSession(us, false);
+
+ return sid;
+ }
+
+ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) {
+ long t1 = System.currentTimeMillis();
+ if (us.currentTablet != null && us.currentTablet.getExtent().equals(keyExtent))
+ return;
+ if (us.currentTablet == null && (us.failures.containsKey(keyExtent) || us.authFailures.containsKey(keyExtent))) {
+ // if there were previous failures, then do not accept additional writes
+ return;
+ }
+
+ try {
+ // if user has no permission to write to this table, add it to
+ // the failures list
+ boolean sameTable = us.currentTablet != null && (us.currentTablet.getExtent().getTableId().equals(keyExtent.getTableId()));
+ String tableId = keyExtent.getTableId().toString();
+ if (sameTable || security.canWrite(us.credentials, tableId, Tables.getNamespaceId(instance, tableId))) {
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = onlineTablets.get(keyExtent);
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ } else {
+ // not serving tablet, so report all mutations as
+ // failures
+ us.failures.put(keyExtent, 0l);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.unknownTabletErrors, 0);
+ }
+ } else {
+ log.warn("Denying access to table " + keyExtent.getTableId() + " for user " + us.credentials.getPrincipal());
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, SecurityErrorCode.PERMISSION_DENIED);
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ } catch (ThriftSecurityException e) {
+ log.error("Denying permission to check user " + us.credentials.getPrincipal() + " with user " + e.getUser(), e);
+ long t2 = System.currentTimeMillis();
+ us.authTimes.addStat(t2 - t1);
+ us.currentTablet = null;
+ us.authFailures.put(keyExtent, e.getCode());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.permissionErrors, 0);
+ return;
+ }
+ }
+
+ @Override
+ public void applyUpdates(TInfo tinfo, long updateID, TKeyExtent tkeyExtent, List<TMutation> tmutations) {
+ UpdateSession us = (UpdateSession) sessionManager.reserveSession(updateID);
+ if (us == null) {
+ throw new RuntimeException("No Such SessionID");
+ }
+
+ try {
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ setUpdateTablet(us, keyExtent);
+
+ if (us.currentTablet != null) {
+ List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
+ for (TMutation tmutation : tmutations) {
+ Mutation mutation = new ServerMutation(tmutation);
+ mutations.add(mutation);
+ us.queuedMutationSize += mutation.numBytes();
+ }
+ if (us.queuedMutationSize > getSystemConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
+ flush(us);
+ }
+ } finally {
+ sessionManager.unreserveSession(us);
+ }
+ }
+
+ private void flush(UpdateSession us) {
+
+ int mutationCount = 0;
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+ Throwable error = null;
+
+ long pt1 = System.currentTimeMillis();
+
+ boolean containsMetadataTablet = false;
+ for (Tablet tablet : us.queuedMutations.keySet())
+ if (tablet.getExtent().isMeta())
+ containsMetadataTablet = true;
+
+ if (!containsMetadataTablet && us.queuedMutations.size() > 0)
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Span prep = Trace.start("prep");
+ try {
+ for (Entry<Tablet,? extends List<Mutation>> entry : us.queuedMutations.entrySet()) {
+
+ Tablet tablet = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+ if (mutations.size() > 0) {
+ try {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.mutationArraySize, mutations.size());
+
+ CommitSession commitSession = tablet.prepareMutationsForCommit(us.cenv, mutations);
+ if (commitSession == null) {
+ if (us.currentTablet == tablet) {
+ us.currentTablet = null;
+ }
+ us.failures.put(tablet.getExtent(), us.successfulCommits.get(tablet));
+ } else {
+ sendables.put(commitSession, mutations);
+ mutationCount += mutations.size();
+ }
+
+ } catch (TConstraintViolationException e) {
+ us.violations.add(e.getViolations());
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.constraintViolations, 0);
+
+ if (e.getNonViolators().size() > 0) {
+ // only log and commit mutations if there were some
+ // that did not
+ // violate constraints... this is what
+ // prepareMutationsForCommit()
+ // expects
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ }
+
+ mutationCount += mutations.size();
+
+ } catch (HoldTimeoutException t) {
+ error = t;
+ log.debug("Giving up on mutations due to a long memory hold time");
+ break;
+ } catch (Throwable t) {
+ error = t;
+ log.error("Unexpected error preparing for commit", error);
+ break;
+ }
+ }
+ }
+ } finally {
+ prep.stop();
+ }
+
+ long pt2 = System.currentTimeMillis();
+ us.prepareTimes.addStat(pt2 - pt1);
+ updateAvgPrepTime(pt2 - pt1, us.queuedMutations.size());
+
+ if (error != null) {
+ for (Entry<CommitSession,List<Mutation>> e : sendables.entrySet()) {
+ e.getKey().abortCommit(e.getValue());
+ }
+ throw new RuntimeException(error);
+ }
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ while (true) {
+ try {
+ long t1 = System.currentTimeMillis();
+
+ logger.logManyTablets(sendables);
+
+ long t2 = System.currentTimeMillis();
+ us.walogTimes.addStat(t2 - t1);
+ updateWalogWriteTime((t2 - t1));
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ wal.stop();
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+
+ Tablet tablet = commitSession.getTablet();
+
+ if (tablet == us.currentTablet) {
+ // because constraint violations may filter out some
+ // mutations, for proper
+ // accounting with the client code, need to increment
+ // the count based
+ // on the original number of mutations from the client
+ // NOT the filtered number
+ us.successfulCommits.increment(tablet, us.queuedMutations.get(tablet).size());
+ }
+ }
+ long t2 = System.currentTimeMillis();
+
+ us.flushTime += (t2 - pt1);
+ us.commitTimes.addStat(t2 - t1);
+
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commit.stop();
+ }
+ } finally {
+ us.queuedMutations.clear();
+ if (us.currentTablet != null) {
+ us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
+ }
+ us.queuedMutationSize = 0;
+ }
+ us.totalUpdates += mutationCount;
+ }
+
+ private void updateWalogWriteTime(long time) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.waLogWriteTime, time);
+ }
+
+ private void updateAvgCommitTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitTime, (long) ((time) / (double) size));
+ }
+
+ private void updateAvgPrepTime(long time, int size) {
+ if (updateMetrics.isEnabled())
+ updateMetrics.add(TabletServerUpdateMetrics.commitPrep, (long) ((time) / (double) size));
+ }
+
+ @Override
+ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDException {
+ UpdateSession us = (UpdateSession) sessionManager.removeSession(updateID);
+ if (us == null) {
+ throw new NoSuchScanIDException();
+ }
+
+ // clients may or may not see data from an update session while
+ // it is in progress, however when the update session is closed
+ // want to ensure that reads wait for the write to finish
+ long opid = writeTracker.startWrite(us.queuedMutations.keySet());
+
+ try {
+ flush(us);
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+
+ log.debug(String.format("UpSess %s %,d in %.3fs, at=[%s] ft=%.3fs(pt=%.3fs lt=%.3fs ct=%.3fs)", TServerUtils.clientAddress.get(), us.totalUpdates,
+ (System.currentTimeMillis() - us.startTime) / 1000.0, us.authTimes.toString(), us.flushTime / 1000.0, us.prepareTimes.getSum() / 1000.0,
+ us.walogTimes.getSum() / 1000.0, us.commitTimes.getSum() / 1000.0));
+ if (us.failures.size() > 0) {
+ Entry<KeyExtent,Long> first = us.failures.entrySet().iterator().next();
+ log.debug(String.format("Failures: %d, first extent %s successful commits: %d", us.failures.size(), first.getKey().toString(), first.getValue()));
+ }
+ List<ConstraintViolationSummary> violations = us.violations.asList();
+ if (violations.size() > 0) {
+ ConstraintViolationSummary first = us.violations.asList().iterator().next();
+ log.debug(String.format("Violations: %d, first %s occurs %d", violations.size(), first.violationDescription, first.numberOfViolatingMutations));
+ }
+ if (us.authFailures.size() > 0) {
+ KeyExtent first = us.authFailures.keySet().iterator().next();
+ log.debug(String.format("Authentication Failures: %d, first %s", us.authFailures.size(), first.toString()));
+ }
+
+ return new UpdateErrors(Translator.translate(us.failures, Translators.KET), Translator.translate(violations, Translators.CVST), Translator.translate(
+ us.authFailures, Translators.KET));
+ }
+
+ @Override
+ public void update(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, TMutation tmutation) throws NotServingTabletException,
+ ConstraintViolationException, ThriftSecurityException {
+
- String tableId = new String(tkeyExtent.getTable(), Constants.UTF8);
++ String tableId = new String(tkeyExtent.getTable(), UTF_8);
+ if (!security.canWrite(credentials, tableId, Tables.getNamespaceId(instance, tableId)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+ Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent));
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!keyExtent.isMeta())
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ long opid = writeTracker.startWrite(TabletType.type(keyExtent));
+
+ try {
+ Mutation mutation = new ServerMutation(tmutation);
+ List<Mutation> mutations = Collections.singletonList(mutation);
+
+ Span prep = Trace.start("prep");
+ CommitSession cs;
+ try {
+ cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, credentials), mutations);
+ } finally {
+ prep.stop();
+ }
+ if (cs == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ while (true) {
+ try {
+ Span wal = Trace.start("wal");
+ try {
+ logger.log(cs, cs.getWALogSeq(), mutation);
+ } finally {
+ wal.stop();
+ }
+ break;
+ } catch (IOException ex) {
+ log.warn(ex, ex);
+ }
+ }
+
+ Span commit = Trace.start("commit");
+ try {
+ cs.commit(mutations);
+ } finally {
+ commit.stop();
+ }
+ } catch (TConstraintViolationException e) {
+ throw new ConstraintViolationException(Translator.translate(e.getViolations().asList(), Translators.CVST));
+ } finally {
+ writeTracker.finishWrite(opid);
+ }
+ }
+
+ private void checkConditions(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs,
+ List<String> symbols) throws IOException {
+ Iterator<Entry<KeyExtent,List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
+
+ CompressedIterators compressedIters = new CompressedIterators(symbols);
+
+ while (iter.hasNext()) {
+ Entry<KeyExtent,List<ServerConditionalMutation>> entry = iter.next();
+ Tablet tablet = onlineTablets.get(entry.getKey());
+
+ if (tablet == null || tablet.isClosed()) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ iter.remove();
+ } else {
+ List<ServerConditionalMutation> okMutations = new ArrayList<ServerConditionalMutation>(entry.getValue().size());
+
+ for (ServerConditionalMutation scm : entry.getValue()) {
+ if (checkCondition(results, cs, compressedIters, tablet, scm))
+ okMutations.add(scm);
+ }
+
+ entry.setValue(okMutations);
+ }
+
+ }
+ }
+
+ boolean checkCondition(ArrayList<TCMResult> results, ConditionalSession cs, CompressedIterators compressedIters, Tablet tablet,
+ ServerConditionalMutation scm) throws IOException {
+ boolean add = true;
+
+ Set<Column> emptyCols = Collections.emptySet();
+
+ for (TCondition tc : scm.getConditions()) {
+
+ Range range;
+ if (tc.hasTimestamp)
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()), tc.getTs());
+ else
+ range = Range.exact(new Text(scm.getRow()), new Text(tc.getCf()), new Text(tc.getCq()), new Text(tc.getCv()));
+
+ IterConfig ic = compressedIters.decompress(tc.iterators);
+
+ Scanner scanner = tablet.createScanner(range, 1, emptyCols, cs.auths, ic.ssiList, ic.ssio, false, cs.interruptFlag);
+
+ try {
+ ScanBatch batch = scanner.read();
+
+ Value val = null;
+
+ for (KVEntry entry2 : batch.results) {
+ val = entry2.getValue();
+ break;
+ }
+
+ if ((val == null ^ tc.getVal() == null) || (val != null && !Arrays.equals(tc.getVal(), val.get()))) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.REJECTED));
+ add = false;
+ break;
+ }
+
+ } catch (TabletClosedException e) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (IterationInterruptedException iie) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ } catch (TooManyFilesException tmfe) {
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ add = false;
+ break;
+ }
+ }
+ return add;
+ }
+
+ private void writeConditionalMutations(Map<KeyExtent,List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession sess) {
+ Set<Entry<KeyExtent,List<ServerConditionalMutation>>> es = updates.entrySet();
+
+ Map<CommitSession,List<Mutation>> sendables = new HashMap<CommitSession,List<Mutation>>();
+
+ boolean sessionCanceled = sess.interruptFlag.get();
+
+ Span prepSpan = Trace.start("prep");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<KeyExtent,List<ServerConditionalMutation>> entry : es) {
+ Tablet tablet = onlineTablets.get(entry.getKey());
+ if (tablet == null || tablet.isClosed() || sessionCanceled) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ try {
+
+ @SuppressWarnings("unchecked")
+ List<Mutation> mutations = (List<Mutation>) (List<? extends Mutation>) entry.getValue();
+ if (mutations.size() > 0) {
+
+ CommitSession cs = tablet.prepareMutationsForCommit(new TservConstraintEnv(security, sess.credentials), mutations);
+
+ if (cs == null) {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
+ } else {
+ for (ServerConditionalMutation scm : entry.getValue())
+ results.add(new TCMResult(scm.getID(), TCMStatus.ACCEPTED));
+ sendables.put(cs, mutations);
+ }
+ }
+ } catch (TConstraintViolationException e) {
+ if (e.getNonViolators().size() > 0) {
+ sendables.put(e.getCommitSession(), e.getNonViolators());
+ for (Mutation m : e.getNonViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.ACCEPTED));
+ }
+
+ for (Mutation m : e.getViolators())
+ results.add(new TCMResult(((ServerConditionalMutation) m).getID(), TCMStatus.VIOLATED));
+ }
+ }
+ }
+
+ long t2 = System.currentTimeMillis();
+ updateAvgPrepTime(t2 - t1, es.size());
+ } finally {
+ prepSpan.stop();
+ }
+
+ Span walSpan = Trace.start("wal");
+ try {
+ while (true && sendables.size() > 0) {
+ try {
+ long t1 = System.currentTimeMillis();
+ logger.logManyTablets(sendables);
+ long t2 = System.currentTimeMillis();
+ updateWalogWriteTime(t2 - t1);
+ break;
+ } catch (IOException ex) {
+ log.warn("logging mutations failed, retrying");
+ } catch (FSError ex) { // happens when DFS is localFS
+ log.warn("logging mutations failed, retrying");
+ } catch (Throwable t) {
+ log.error("Unknown exception logging mutations, counts for mutations in flight not decremented!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ } finally {
+ walSpan.stop();
+ }
+
+ Span commitSpan = Trace.start("commit");
+ try {
+ long t1 = System.currentTimeMillis();
+ for (Entry<CommitSession,? extends List<Mutation>> entry : sendables.entrySet()) {
+ CommitSession commitSession = entry.getKey();
+ List<Mutation> mutations = entry.getValue();
+
+ commitSession.commit(mutations);
+ }
+ long t2 = System.currentTimeMillis();
+ updateAvgCommitTime(t2 - t1, sendables.size());
+ } finally {
+ commitSpan.stop();
+ }
+
+ }
+
+ private Map<KeyExtent,List<ServerConditionalMutation>> conditionalUpdate(ConditionalSession cs, Map<KeyExtent,List<ServerConditionalMutation>> updates,
+ ArrayList<TCMResult> results, List<String> symbols) throws IOException {
+ // sort each list of mutations, this is done to avoid deadlock and doing seeks in order is more efficient and detect duplicate rows.
+ ConditionalMutationSet.sortConditionalMutations(updates);
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = new HashMap<KeyExtent,List<ServerConditionalMutation>>();
+
+ // can not process two mutations for the same row, because one will not see what the other writes
+ ConditionalMutationSet.deferDuplicatesRows(updates, deferred);
+
+ // get as many locks as possible w/o blocking... defer any rows that are locked
+ List<RowLock> locks = rowLocks.acquireRowlocks(updates, deferred);
+ try {
+ Span checkSpan = Trace.start("Check conditions");
+ try {
+ checkConditions(updates, results, cs, symbols);
+ } finally {
+ checkSpan.stop();
+ }
+
+ Span updateSpan = Trace.start("apply conditional mutations");
+ try {
+ writeConditionalMutations(updates, results, cs);
+ } finally {
+ updateSpan.stop();
+ }
+ } finally {
+ rowLocks.releaseRowLocks(locks);
+ }
+ return deferred;
+ }
+
+ @Override
+ public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials credentials, List<ByteBuffer> authorizations, String tableId)
+ throws ThriftSecurityException, TException {
+
+ Authorizations userauths = null;
+ if (!security.canConditionallyUpdate(credentials, tableId, Tables.getNamespaceId(instance, tableId), authorizations))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ userauths = security.getUserAuthorizations(credentials);
+ for (ByteBuffer auth : authorizations)
+ if (!userauths.contains(ByteBufferUtil.toBytes(auth)))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
+
+ ConditionalSession cs = new ConditionalSession();
+ cs.auths = new Authorizations(authorizations);
+ cs.credentials = credentials;
+ cs.tableId = tableId;
+ cs.interruptFlag = new AtomicBoolean();
+
+ long sid = sessionManager.createSession(cs, false);
+ return new TConditionalSession(sid, lockID, sessionManager.getMaxIdleTime());
+ }
+
+ @Override
+ public List<TCMResult> conditionalUpdate(TInfo tinfo, long sessID, Map<TKeyExtent,List<TConditionalMutation>> mutations, List<String> symbols)
+ throws NoSuchScanIDException, TException {
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.reserveSession(sessID);
+
+ if (cs == null || cs.interruptFlag.get())
+ throw new NoSuchScanIDException();
+
+ if (!cs.tableId.equals(MetadataTable.ID) && !cs.tableId.equals(RootTable.ID))
+ TabletServer.this.resourceManager.waitUntilCommitsAreEnabled();
+
+ Text tid = new Text(cs.tableId);
+ long opid = writeTracker.startWrite(TabletType.type(new KeyExtent(tid, null, null)));
+
+ try {
+ Map<KeyExtent,List<ServerConditionalMutation>> updates = Translator.translate(mutations, Translators.TKET,
+ new Translator.ListTranslator<TConditionalMutation,ServerConditionalMutation>(ServerConditionalMutation.TCMT));
+
+ for (KeyExtent ke : updates.keySet())
+ if (!ke.getTableId().equals(tid))
+ throw new IllegalArgumentException("Unexpected table id " + tid + " != " + ke.getTableId());
+
+ ArrayList<TCMResult> results = new ArrayList<TCMResult>();
+
+ Map<KeyExtent,List<ServerConditionalMutation>> deferred = conditionalUpdate(cs, updates, results, symbols);
+
+ while (deferred.size() > 0) {
+ deferred = conditionalUpdate(cs, deferred, results, symbols);
+ }
+
+ return results;
+ } catch (IOException ioe) {
+ throw new TException(ioe);
+ } finally {
+ writeTracker.finishWrite(opid);
+ sessionManager.unreserveSession(sessID);
+ }
+ }
+
+ @Override
+ public void invalidateConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ // this method should wait for any running conditional update to complete
+ // after this method returns a conditional update should not be able to start
+
+ ConditionalSession cs = (ConditionalSession) sessionManager.getSession(sessID);
+ if (cs != null)
+ cs.interruptFlag.set(true);
+
+ cs = (ConditionalSession) sessionManager.reserveSession(sessID, true);
+ if (cs != null)
+ sessionManager.removeSession(sessID, true);
+ }
+
+ @Override
+ public void closeConditionalUpdate(TInfo tinfo, long sessID) throws TException {
+ sessionManager.removeSession(sessID, false);
+ }
+
+ @Override
+ public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException,
+ ThriftSecurityException {
+
+ String tableId = new String(ByteBufferUtil.toBytes(tkeyExtent.table));
+ String namespaceId;
+ try {
+ namespaceId = Tables.getNamespaceId(instance, tableId);
+ } catch (IllegalArgumentException ex) {
+ // table does not exist, try to educate the client
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (!security.canSplitTablet(credentials, tableId, namespaceId))
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+
+ KeyExtent keyExtent = new KeyExtent(tkeyExtent);
+
+ Tablet tablet = onlineTablets.get(keyExtent);
+ if (tablet == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+
+ if (keyExtent.getEndRow() == null || !keyExtent.getEndRow().equals(ByteBufferUtil.toText(splitPoint))) {
+ try {
+ if (TabletServer.this.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
+ throw new NotServingTabletException(tkeyExtent);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to split " + keyExtent, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public TabletServerStatus getTabletServerStatus(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException, TException {
+ return getStats(sessionManager.getActiveScansPerTable());
+ }
+
+ @Override
+ public List<TabletStats> getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) throws ThriftSecurityException, TException {
+ TreeMap<KeyExtent,Tablet> onlineTabletsCopy;
+ synchronized (onlineTablets) {
+ onlineTabletsCopy = new TreeMap<KeyExtent,Tablet>(onlineTablets);
+ }
+ List<TabletStats> result = new ArrayList<TabletStats>();
+ Text text = new Text(tableId);
+ KeyExtent start = new KeyExtent(text, new Text(), null);
+ for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.tailMap(start).entrySet()) {
+ KeyExtent ke = entry.getKey();
+ if (ke.getTableId().compareTo(text) == 0) {
+ Tablet tablet = entry.getValue();
+ TabletStats stats = tablet.timer.getTabletStats();
+ stats.extent = ke.toThrift();
+ stats.ingestRate = tablet.ingestRate();
+ stats.queryRate = tablet.queryRate();
+ stats.splitCreationTime = tablet.getSplitCreationTime();
+ stats.numEntries = tablet.getNumEntries();
+ result.add(stats);
+ }
+ }
+ return result;
+ }
+
+ private ZooCache masterLockCache = new ZooCache();
+
+ private void checkPermission(TCredentials credentials, String lock, final String request) throws ThriftSecurityException {
+ boolean fatal = false;
+ try {
+ log.debug("Got " + request + " message from user: " + credentials.getPrincipal());
+ if (!security.canPerformSystemActions(credentials)) {
+ log.warn("Got " + request + " message from user: " + credentials.getPrincipal());
+ throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
+ }
+ } catch (ThriftSecurityException e) {
+ log.warn("Got " + request + " message from unauthenticatable user: " + e.getUser());
+ if (SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName())) {
+ log.fatal("Got message from a service with a mismatched configuration. Please ensure a compatible configu
<TRUNCATED>