You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2017/01/25 17:54:37 UTC
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8
Conflicts:
server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b903766e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b903766e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b903766e
Branch: refs/heads/master
Commit: b903766ec14b36ac4185ab658d2155970cc7ffb4
Parents: 95be9f3 df400c5
Author: Keith Turner <kt...@apache.org>
Authored: Wed Jan 25 12:42:41 2017 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Wed Jan 25 12:42:41 2017 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/fate/AdminUtil.java | 104 +++++++++++--
.../accumulo/cluster/AccumuloCluster.java | 6 +
.../standalone/StandaloneAccumuloCluster.java | 12 ++
.../impl/MiniAccumuloClusterImpl.java | 8 +
.../accumulo/master/FateServiceHandler.java | 2 +-
.../accumulo/master/tableOps/DeleteTable.java | 49 +++----
.../apache/accumulo/master/tableOps/Utils.java | 13 ++
.../test/functional/BackupMasterIT.java | 7 +-
.../functional/ConcurrentDeleteTableIT.java | 147 +++++++++++++++++++
9 files changed, 307 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index a3180a7,79ad527..829b321
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -110,12 -111,12 +111,13 @@@ import org.slf4j.LoggerFactory
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
+ import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Uninterruptibles;
/**
- * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
- * code against a real Accumulo instance. Its much more accurate for testing than {@link org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower.
+ * This class provides the backing implementation for {@link MiniAccumuloCluster}, and may contain features for internal testing which have not yet been
+ * promoted to the public API. It's best to use {@link MiniAccumuloCluster} whenever possible. Use of this class risks API breakage between versions.
*
* @since 1.6.0
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
----------------------------------------------------------------------
diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
index 2baf7ac,9b921e2..aa02559
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java
@@@ -117,6 -116,19 +117,19 @@@ public class Utils
return 100;
}
+ public static String getNamespaceId(Instance instance, String tableId, TableOperation op) throws Exception {
+ try {
+ return Tables.getNamespaceId(instance, tableId);
+ } catch (RuntimeException e) {
+ // see if this was caused because the table does not exists
+ IZooReaderWriter zk = ZooReaderWriter.getInstance();
+ if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId))
- throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist");
++ throw new AcceptableThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist");
+ else
+ throw e;
+ }
+ }
+
public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException {
Instance instance = HdfsZooInstance.getInstance();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
index d8979db,0000000..12336bd
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java
@@@ -1,68 -1,0 +1,71 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.Collections;
+import java.util.List;
+
++import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.util.UtilWaitThread;
- import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
++import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.master.Master;
++import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
+import org.junit.Test;
+
+public class BackupMasterIT extends ConfigurableMacBase {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 120;
+ }
+
+ @Test
+ public void test() throws Exception {
+ // wait for master
+ UtilWaitThread.sleep(1000);
+ // create a backup
+ Process backup = exec(Master.class);
+ try {
- ZooReaderWriter writer = new ZooReaderWriter(cluster.getZooKeepers(), 30 * 1000, "digest", "accumulo:DONTTELL".getBytes());
++ String secret = getCluster().getSiteConfiguration().get(Property.INSTANCE_SECRET);
++ IZooReaderWriter writer = new ZooReaderWriterFactory().getZooReaderWriter(cluster.getZooKeepers(), 30 * 1000, secret);
+ String root = "/accumulo/" + getConnector().getInstance().getInstanceID();
+ List<String> children = Collections.emptyList();
+ // wait for 2 lock entries
+ do {
+ UtilWaitThread.sleep(100);
+ children = writer.getChildren(root + "/masters/lock");
+ } while (children.size() != 2);
+ Collections.sort(children);
+ // wait for the backup master to learn to be the backup
+ UtilWaitThread.sleep(1000);
+ // generate a false zookeeper event
+ String lockPath = root + "/masters/lock/" + children.get(0);
+ byte data[] = writer.getData(lockPath, null);
+ writer.getZooKeeper().setData(lockPath, data, -1);
+ // let it propagate
+ UtilWaitThread.sleep(500);
+ // kill the master by removing its lock
+ writer.recursiveDelete(lockPath, NodeMissingPolicy.FAIL);
+ // ensure the backup becomes the master
+ getConnector().tableOperations().create(getUniqueNames(1)[0]);
+ } finally {
+ backup.destroy();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/test/src/main/java/org/apache/accumulo/test/functional/ConcurrentDeleteTableIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ConcurrentDeleteTableIT.java
index 0000000,0000000..3f7a305
new file mode 100644
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ConcurrentDeleteTableIT.java
@@@ -1,0 -1,0 +1,147 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.accumulo.test.functional;
++
++import java.util.ArrayList;
++import java.util.List;
++import java.util.Random;
++import java.util.TreeSet;
++import java.util.concurrent.CountDownLatch;
++import java.util.concurrent.ExecutorService;
++import java.util.concurrent.Executors;
++import java.util.concurrent.Future;
++
++import org.apache.accumulo.core.Constants;
++import org.apache.accumulo.core.client.AccumuloException;
++import org.apache.accumulo.core.client.AccumuloSecurityException;
++import org.apache.accumulo.core.client.BatchWriter;
++import org.apache.accumulo.core.client.BatchWriterConfig;
++import org.apache.accumulo.core.client.Connector;
++import org.apache.accumulo.core.client.Instance;
++import org.apache.accumulo.core.client.MutationsRejectedException;
++import org.apache.accumulo.core.client.TableNotFoundException;
++import org.apache.accumulo.core.conf.Property;
++import org.apache.accumulo.core.data.Mutation;
++import org.apache.accumulo.core.security.Authorizations;
++import org.apache.accumulo.core.zookeeper.ZooUtil;
++import org.apache.accumulo.fate.AdminUtil;
++import org.apache.accumulo.fate.AdminUtil.FateStatus;
++import org.apache.accumulo.fate.ZooStore;
++import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
++import org.apache.accumulo.harness.AccumuloClusterHarness;
++import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
++import org.apache.hadoop.io.Text;
++import org.apache.zookeeper.KeeperException;
++import org.junit.Assert;
++import org.junit.Test;
++
++public class ConcurrentDeleteTableIT extends AccumuloClusterHarness {
++
++ @Test
++ public void testConcurrentDeleteTablesOps() throws Exception {
++ final Connector c = getConnector();
++ String[] tables = getUniqueNames(2);
++
++ TreeSet<Text> splits = new TreeSet<>();
++
++ for (int i = 0; i < 1000; i++) {
++ Text split = new Text(String.format("%09x", i * 100000));
++ splits.add(split);
++ }
++
++ ExecutorService es = Executors.newFixedThreadPool(20);
++
++ int count = 0;
++ for (final String table : tables) {
++ c.tableOperations().create(table);
++ c.tableOperations().addSplits(table, splits);
++ writeData(c, table);
++ if (count == 1) {
++ c.tableOperations().flush(table, null, null, true);
++ }
++ count++;
++
++ final CountDownLatch cdl = new CountDownLatch(20);
++
++ List<Future<?>> futures = new ArrayList<>();
++
++ for (int i = 0; i < 20; i++) {
++ Future<?> future = es.submit(new Runnable() {
++
++ @Override
++ public void run() {
++ try {
++ cdl.countDown();
++ cdl.await();
++ c.tableOperations().delete(table);
++ } catch (TableNotFoundException e) {
++ // expected
++ } catch (InterruptedException | AccumuloException | AccumuloSecurityException e) {
++ throw new RuntimeException(e);
++ }
++ }
++ });
++
++ futures.add(future);
++ }
++
++ for (Future<?> future : futures) {
++ future.get();
++ }
++
++ try {
++ c.createScanner(table, Authorizations.EMPTY);
++ Assert.fail("Expected table " + table + " to be gone.");
++ } catch (TableNotFoundException tnfe) {
++ // expected
++ }
++
++ FateStatus fateStatus = getFateStatus();
++
++ // ensure there are no dangling locks... before ACCUMULO-4575 was fixed concurrent delete tables could fail and leave dangling locks.
++ Assert.assertEquals(0, fateStatus.getDanglingHeldLocks().size());
++ Assert.assertEquals(0, fateStatus.getDanglingWaitingLocks().size());
++ }
++
++ es.shutdown();
++ }
++
++ private FateStatus getFateStatus() throws KeeperException, InterruptedException {
++ Instance instance = getConnector().getInstance();
++ AdminUtil<String> admin = new AdminUtil<>(false);
++ String secret = getCluster().getSiteConfiguration().get(Property.INSTANCE_SECRET);
++ IZooReaderWriter zk = new ZooReaderWriterFactory().getZooReaderWriter(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut(), secret);
++ ZooStore<String> zs = new ZooStore<String>(ZooUtil.getRoot(instance) + Constants.ZFATE, zk);
++ FateStatus fateStatus = admin.getStatus(zs, zk, ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, null, null);
++ return fateStatus;
++ }
++
++ private void writeData(Connector c, String table) throws TableNotFoundException, MutationsRejectedException {
++ BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
++ try {
++ Random rand = new Random();
++ for (int i = 0; i < 1000; i++) {
++ Mutation m = new Mutation(String.format("%09x", rand.nextInt(100000 * 1000)));
++ m.put("m", "order", "" + i);
++ bw.addMutation(m);
++ }
++ } finally {
++ bw.close();
++ }
++ }
++}